funboost 43.7__tar.gz → 43.9__tar.gz

{funboost-43.7/funboost.egg-info → funboost-43.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.7
+Version: 43.9
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.7 → funboost-43.9}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "43.7"
+__version__ = "43.9"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-43.7 → funboost-43.9}/funboost/consumers/base_consumer.py

@@ -33,7 +33,7 @@ import nb_log
 from funboost.core.current_task import funboost_current_task, get_current_taskid
 from funboost.core.loggers import develop_logger
 
-from funboost.core.func_params_model import BoosterParams, PublisherParams
+from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
 from funboost.core.task_id_logger import TaskIdLogger
 from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, CompatibleLogger,
                     LoggerMixinDefaultWithFileHandler, stdout_write, is_main_process,
@@ -49,7 +49,7 @@ from funboost.concurrent_pool.single_thread_executor import SoloExecutor
 
 from funboost.core.function_result_status_saver import ResultPersistenceHelper, FunctionResultStatus, RunStatus
 
-from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time
+from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, generate_task_id
 
 from funboost.concurrent_pool.async_helper import simple_run_in_executor
 from funboost.concurrent_pool.async_pool_executor import AsyncPoolExecutor
@@ -215,13 +215,15 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._consuming_function_is_asyncio = inspect.iscoroutinefunction(self.consuming_function)
         self.custom_init()
         # develop_logger.warning(consumer_params._log_filename)
-        self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
-                                                broker_kind=self.BROKER_KIND, log_level=consumer_params.log_level,
-                                                logger_prefix=consumer_params.logger_prefix,
-                                                create_logger_file=consumer_params.create_logger_file,
-                                                log_filename=consumer_params.log_filename,
-                                                logger_name=consumer_params.logger_name,
-                                                broker_exclusive_config=self.consumer_params.broker_exclusive_config)
+        # self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
+        #                                         broker_kind=self.BROKER_KIND, log_level=consumer_params.log_level,
+        #                                         logger_prefix=consumer_params.logger_prefix,
+        #                                         create_logger_file=consumer_params.create_logger_file,
+        #                                         log_filename=consumer_params.log_filename,
+        #                                         logger_name=consumer_params.logger_name,
+        #                                         broker_exclusive_config=self.consumer_params.broker_exclusive_config)
+        self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, consumer_params)
+        # print(self.publisher_params)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
         atexit.register(self.join_shedual_task_thread)
@@ -381,6 +383,33 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         raise NotImplementedError
 
+    def __auto_fill_msg(self, msg: dict):
+        """填充消息,消息没有使用funboost来发送,并且没有extra相关字段时候"""
+        """ 一般消息至少包含这样
+       {
+  "a": 42,
+  "b": 84,
+  "extra": {
+    "task_id": "queue_2_result:9b79a372-f765-4a33-8639-9d15d7a95f61",
+    "publish_time": 1701687443.3596,
+    "publish_time_format": "2023-12-04 18:57:23"
+  }
+}
+        """
+        """
+        extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
+                        'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
+        """
+        if 'extra' not in msg:
+            msg['extra'] = {}
+        extra = msg['extra']
+        if 'task_id' not in extra:
+            extra['task_id'] = generate_task_id(self._queue_name)
+        if 'publish_time' not in extra:
+            extra['publish_time'] = round(time.time(), 4)
+        if 'publish_time_format':
+            extra['publish_time_format'] = time.strftime('%Y-%m-%d %H:%M:%S')
+
     def _submit_task(self, kw):
         while 1:  # 这一块的代码为支持暂停消费。
             # print(self._pause_flag)
@@ -396,7 +425,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             self._requeue(kw)
             time.sleep(self.time_interval_for_check_do_not_run_time)
             return
-
+        self.__auto_fill_msg(kw['body'])
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
                 function_only_params):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。

{funboost-43.7 → funboost-43.9}/funboost/core/booster.py

@@ -155,7 +155,9 @@ class Booster:
                       file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
                       invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
                       python_interpreter='python3',
-                      process_num=1):
+                      process_num=1,
+                      pkey_file_path=None,
+                      ):
         """
         入参见 fabric_deploy 函数。这里重复入参是为了代码在pycharm补全提示。
         """

{funboost-43.7 → funboost-43.9}/funboost/core/fabric_deploy_helper.py

@@ -1,17 +1,12 @@
 # noinspection PyDefaultArgument
-import re
 import sys
 import threading
 import time
-import os
-import typing
 from pathlib import Path
 from fabric2 import Connection
 from nb_libs.path_helper import PathHelper
-# from funboost.core.booster import Booster
 from funboost.utils.paramiko_util import ParamikoFolderUploader
 
-# import nb_log
 from funboost.core.loggers import get_funboost_file_logger
 from funboost.core.booster import Booster
 
@@ -19,9 +14,6 @@ logger = get_funboost_file_logger(__name__)
 
 
 # noinspection PyDefaultArgument
-
-
-
 def fabric_deploy(booster: Booster, host, port, user, password,
                   path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
                   file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
@@ -29,7 +21,9 @@ def fabric_deploy(booster: Booster, host, port, user, password,
                   file_volume_limit=1000 * 1000, sftp_log_level=20, extra_shell_str='',
                   invoke_runner_kwargs={'hide': None, 'pty': True, 'warn': False},
                   python_interpreter='python3',
-                  process_num=1):
+                  process_num=1,
+                  pkey_file_path=None,
+                  ):
     """
     不依赖阿里云codepipeline 和任何运维发布管理工具，只需要在python代码层面就能实现多机器远程部署。
     这实现了函数级别的精确部署，而非是部署一个 .py的代码，远程部署一个函数实现难度比远程部署一个脚本更高一点，部署更灵活。
@@ -65,6 +59,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
                                  pty 的意思是，远程机器的部署的代码进程是否随着当前脚本的结束而结束。如果为True，本机代码结束远程进程就会结束。如果为False，即使本机代码被关闭结束，远程机器还在运行代码。
                                  warn 的意思是如果远程机器控制台返回了异常码本机代码是否立即退出。warn为True这只是警告一下，warn为False,远程机器返回异常code码则本机代码直接终止退出。
     :param process_num:启动几个进程，要达到最大cpu性能就开启cpu核数个进程就可以了。每个进程内部都有任务函数本身指定的并发方式和并发数量，所以是多进程+线程/协程。
+    :param pkey_file_path: 私钥文件路径，如果设置了这个参数，则使用ssh私钥登录远程机器，如果没设置，则使用密码登录。
     :return:
 
 
@@ -80,7 +75,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
     func_name = booster.consuming_function.__name__
 
     """以下这种是为了兼容 函数没有@boost,而是使用 boosterxx = BoostersManager.build_booster() 来创建的booster. 下面的 python_exec_str 中需要用到 func_name 
-    也可以远程时候使用 BoostersManager.get_booster(queue_name),然后启动消费.  因为import模块后,就能启动了.
+    也可以远程时候使用 BoostersManager.get_booster(queue_name),然后启动消费.  因为import模块后,就注册booster信息到BoostersManager,所以能启动了.
     """
     module_obj = PathHelper(sys._getframe(2).f_code.co_filename).import_as_module()  # noqa
     for var_name,var_value in module_obj.__dict__.items():
@@ -99,7 +94,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
         t_start = time.perf_counter()
         uploader = ParamikoFolderUploader(host, port, user, password, python_proj_dir, remote_dir,
                                           path_pattern_exluded_tuple, file_suffix_tuple_exluded,
-                                          only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level)
+                                          only_upload_within_the_last_modify_time, file_volume_limit, sftp_log_level, pkey_file_path)
         uploader.upload()
         logger.info(f'上传 本地文件夹代码 {python_proj_dir}  上传到远程 {host} 的 {remote_dir} 文件夹耗时 {round(time.perf_counter() - t_start, 3)} 秒')
         # conn.run(f'''export PYTHONPATH={remote_dir}:$PYTHONPATH''')

{funboost-43.7 → funboost-43.9}/funboost/core/func_params_model.py

@@ -81,6 +81,14 @@ class BaseJsonAbleModel(BaseModel):
         # allow_mutation = False
         extra = "forbid"
 
+    @staticmethod
+    def init_by_another_model(model_type: typing.Type[BaseModel], modelx: BaseModel):
+        init_dict = {}
+        for k, v in modelx.dict().items():
+            if k in model_type.__fields__.keys():
+                init_dict[k] = v
+        return model_type(**init_dict)
+
 
 class FunctionResultStatusPersistanceConfig(BaseJsonAbleModel):
     is_save_status: bool  # 是否保存函数的运行状态信息
@@ -175,6 +183,8 @@ class BoosterParams(BaseJsonAbleModel):
     broker_exclusive_config: dict = {}  # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置，不是所有中间件都兼容的配置，因为框架支持30种消息队列，消息队列不仅仅是一般的先进先出queue这么简单的概念，
     # 例如kafka支持消费者组，rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制，有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义，可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
 
+    should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
+
     auto_generate_info: dict = {}  # 自动生成的信息,不需要用户主动传参.
 
     @root_validator(skip_on_failure=True)
@@ -194,7 +204,7 @@ class BoosterParams(BaseJsonAbleModel):
         # if not set(values.keys()).issubset(set(BoosterParams.__fields__.keys())):
         #     raise ValueError(f'{cls.__name__} 的字段包含了父类 BoosterParams 不存在的字段')
         for k in values.keys():
-            if k not in BoosterParams.model_fields.keys():
+            if k not in BoosterParams.__fields__.keys():
                 raise ValueError(f'{cls.__name__} 的字段新增了父类 BoosterParams 不存在的字段 "{k}"')  # 使 BoosterParams的子类,不能增加字段,只能覆盖字段.
         return values
 
@@ -255,6 +265,7 @@ class PublisherParams(BaseJsonAbleModel):
     consuming_function: typing.Callable = None  # consuming_function 作用是 inspect 模块获取函数的入参信息
     broker_kind: str = None
     broker_exclusive_config: dict = {}
+    should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
 
 
 if __name__ == '__main__':

{funboost-43.7 → funboost-43.9}/funboost/core/helper_funs.py

@@ -1,5 +1,6 @@
 import copy
 import time
+import uuid
 
 
 def get_publish_time(paramsx: dict):
@@ -42,4 +43,7 @@ def _try_get_user_funboost_common_config(funboost_common_conf_field:str):
         return getattr(funboost_config.FunboostCommonConfig,funboost_common_conf_field)
     except Exception as e:
         print(e)
-        return None
+        return None
+
+def generate_task_id(queue_name:str):
+    return f'{queue_name}_result:{uuid.uuid4()}'

{funboost-43.7 → funboost-43.9}/funboost/core/lazy_impoter.py

@@ -79,6 +79,20 @@ class EventletImporter:
         self.patcher = patcher
         self.Timeout = Timeout
 
+@singleton
+class PeeweeImporter:
+    def __init__(self):
+        '''pip install peewee == 3.17'''
+        from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
+        from playhouse.shortcuts import model_to_dict, dict_to_model
+        self.ModelSelect = ModelSelect
+        self.Model = Model
+        self.BigAutoField = BigAutoField
+        self.CharField = CharField
+        self.DateTimeField =DateTimeField
+        self.MySQLDatabase = MySQLDatabase
+        self.model_to_dict = model_to_dict
+        self.dict_to_model =dict_to_model
 
 if __name__ == '__main__':
     for i in range(10000):

{funboost-43.7 → funboost-43.9}/funboost/publishers/base_publisher.py

@@ -19,6 +19,7 @@ import amqpstorm
 
 import nb_log
 from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
+from funboost.core.helper_funs import generate_task_id
 from funboost.core.loggers import develop_logger
 
 from pikav1.exceptions import AMQPError as PikaAMQPError
@@ -188,9 +189,9 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         if 'extra' in msg:
             msg_function_kw.pop('extra')
             raw_extra = msg['extra']
-        if self.publish_params_checker:
+        if self.publish_params_checker and self.publisher_params.should_check_publish_func_params:
             self.publish_params_checker.check_params(msg_function_kw)
-        task_id = task_id or f'{self._queue_name}_result:{uuid.uuid4()}'
+        task_id = task_id or generate_task_id(self._queue_name)
         extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
                         'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
         if priority_control_config:

{funboost-43.7 → funboost-43.9}/funboost/queues/peewee_queue.py

@@ -1,14 +1,14 @@
 import datetime
 import time
-
-from peewee import ModelSelect
-from playhouse.shortcuts import model_to_dict, dict_to_model
+#
+# from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
+# from playhouse.shortcuts import model_to_dict, dict_to_model
 
 # from nb_log import LoggerMixin, LoggerLevelSetterMixin
 from funboost.core.loggers import LoggerLevelSetterMixin,FunboostFileLoggerMixin
 from funboost.funboost_config_deafult import BrokerConnConfig
-from peewee import *
-
+# from peewee import *
+from funboost.core.lazy_impoter import PeeweeImporter
 
 
 class TaskStatus:
@@ -30,13 +30,13 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
         self._create_table()
 
     def _create_table(self):
-        class FunboostMessage(Model):
+        class FunboostMessage(PeeweeImporter().Model):
             """数据库的一行模拟一条消息"""
-            job_id = BigAutoField(primary_key=True, )
-            body = CharField(max_length=10240, null=False)
-            publish_timestamp = DateTimeField(default=datetime.datetime.now)
-            status = CharField(max_length=40, null=False)
-            consume_start_timestamp = DateTimeField(default=None, null=True)
+            job_id = PeeweeImporter().BigAutoField(primary_key=True, )
+            body = PeeweeImporter().CharField(max_length=10240, null=False)
+            publish_timestamp = PeeweeImporter().DateTimeField(default=datetime.datetime.now)
+            status = PeeweeImporter().CharField(max_length=40, null=False)
+            consume_start_timestamp = PeeweeImporter().DateTimeField(default=None, null=True)
 
             class Meta:
                 db_table = self.queue_name
@@ -47,7 +47,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
                     passwd=BrokerConnConfig.MYSQL_PASSWORD,
                     database=BrokerConnConfig.MYSQL_DATABASE,
                 )
-                database = MySQLDatabase(**conn_params)
+                database = PeeweeImporter().MySQLDatabase(**conn_params)
 
         FunboostMessage.create_table()
         self.FunboostMessage = FunboostMessage
@@ -68,7 +68,7 @@ class PeeweeQueue(FunboostFileLoggerMixin, LoggerLevelSetterMixin):
             # print(ret)
             if len(ret) == 1:
                 row_obj = ret[0]
-                row = model_to_dict(row_obj)
+                row = PeeweeImporter().model_to_dict(row_obj)
                 self.FunboostMessage.update(status=TaskStatus.PENGDING, consume_start_timestamp=datetime.datetime.now()
                                             ).where(self.FunboostMessage.job_id == row['job_id']).execute()
                 return row

{funboost-43.7 → funboost-43.9}/funboost/utils/paramiko_util.py

@@ -16,7 +16,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
                  path_pattern_exluded_tuple=('/.git/', '/.idea/', '/dist/', '/build/'),
                  file_suffix_tuple_exluded=('.pyc', '.log', '.gz'),
                  only_upload_within_the_last_modify_time=3650 * 24 * 60 * 60,
-                 file_volume_limit=1000 * 1000, sftp_log_level=20):
+                 file_volume_limit=1000 * 1000, sftp_log_level=20, pkey_file_path=None):
         """
 
         :param host:
@@ -30,6 +30,7 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
         :param only_upload_within_the_last_modify_time: 仅仅上传最近多少天修改的文件
         :param file_volume_limit: 大于这个体积的不上传，单位b。
         :param sftp_log_level:日志级别
+        :param pkey_file_path: 私钥文件路径，如果设置了这个，那么使用私钥登录。
         """
         self._host = host
         self._port = port
@@ -48,7 +49,11 @@ class ParamikoFolderUploader(LoggerMixin, LoggerLevelSetterMixin):
         self._file_volume_limit = file_volume_limit
 
         t = paramiko.Transport((host, port))
-        t.connect(username=user, password=password)
+        if pkey_file_path is not None and os.path.exists(pkey_file_path):
+            pkey = paramiko.RSAKey.from_private_key_file(open(pkey_file_path))
+            t.connect(username=user, pkey=pkey)
+        else:
+            t.connect(username=user, password=password)
         self.sftp = paramiko.SFTPClient.from_transport(t)
 
         ssh = paramiko.SSHClient()

{funboost-43.7 → funboost-43.9/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.7
+Version: 43.9
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.7 → funboost-43.9}/funboost.egg-info/requires.txt

@@ -16,7 +16,6 @@ kafka-python==2.0.2
 requests
 gnsq==1.0.1
 psutil
-peewee==3.15.1
 apscheduler==3.10.1
 pikav0
 pikav1

{funboost-43.7 → funboost-43.9}/setup.py

@@ -85,7 +85,7 @@ setup(
         'psutil',
         # 'sqlalchemy==1.3.10',
         # 'sqlalchemy_utils==0.36.1',    # 用户使用数据库作为消息队列时候，自行安装，不自动安装这个包。也可以使用kombu中间件的sqlalchemy模式来操作数据库或者peewee操作。
-        'peewee==3.15.1',
+        # 'peewee==3.17.3',    # 'peewee==3.15.1',  # 惰性安装
         'apscheduler==3.10.1',
         'pikav0',
         'pikav1',