funboost 43.8__tar.gz → 44.0__tar.gz

{funboost-43.8/funboost.egg-info → funboost-44.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.8
+Version: 44.0
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.8 → funboost-44.0}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "43.8"
+__version__ = "44.0"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-43.8 → funboost-44.0}/funboost/consumers/base_consumer.py

@@ -33,7 +33,7 @@ import nb_log
 from funboost.core.current_task import funboost_current_task, get_current_taskid
 from funboost.core.loggers import develop_logger
 
-from funboost.core.func_params_model import BoosterParams, PublisherParams
+from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
 from funboost.core.task_id_logger import TaskIdLogger
 from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, CompatibleLogger,
                     LoggerMixinDefaultWithFileHandler, stdout_write, is_main_process,
@@ -49,7 +49,7 @@ from funboost.concurrent_pool.single_thread_executor import SoloExecutor
 
 from funboost.core.function_result_status_saver import ResultPersistenceHelper, FunctionResultStatus, RunStatus
 
-from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time
+from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, generate_task_id
 
 from funboost.concurrent_pool.async_helper import simple_run_in_executor
 from funboost.concurrent_pool.async_pool_executor import AsyncPoolExecutor
@@ -215,13 +215,15 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._consuming_function_is_asyncio = inspect.iscoroutinefunction(self.consuming_function)
         self.custom_init()
         # develop_logger.warning(consumer_params._log_filename)
-        self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
-                                                broker_kind=self.BROKER_KIND, log_level=consumer_params.log_level,
-                                                logger_prefix=consumer_params.logger_prefix,
-                                                create_logger_file=consumer_params.create_logger_file,
-                                                log_filename=consumer_params.log_filename,
-                                                logger_name=consumer_params.logger_name,
-                                                broker_exclusive_config=self.consumer_params.broker_exclusive_config)
+        # self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
+        #                                         broker_kind=self.BROKER_KIND, log_level=consumer_params.log_level,
+        #                                         logger_prefix=consumer_params.logger_prefix,
+        #                                         create_logger_file=consumer_params.create_logger_file,
+        #                                         log_filename=consumer_params.log_filename,
+        #                                         logger_name=consumer_params.logger_name,
+        #                                         broker_exclusive_config=self.consumer_params.broker_exclusive_config)
+        self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, consumer_params)
+        # print(self.publisher_params)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
         atexit.register(self.join_shedual_task_thread)
@@ -381,6 +383,34 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         raise NotImplementedError
 
+    def _auto_fill_msg(self, msg: dict):
+        """填充消息,消息没有使用funboost来发送,并且没有extra相关字段时候"""
+        """ 一般消息至少包含这样
+        {
+          "a": 42,
+          "b": 84,
+          "extra": {
+            "task_id": "queue_2_result:9b79a372-f765-4a33-8639-9d15d7a95f61",
+            "publish_time": 1701687443.3596,
+            "publish_time_format": "2023-12-04 18:57:23"
+          }
+        }
+        """
+
+        """
+        extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
+                        'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
+        """
+        if 'extra' not in msg:
+            msg['extra'] = {'is_auto_fill_extra': True}
+        extra = msg['extra']
+        if 'task_id' not in extra:
+            extra['task_id'] = generate_task_id(self._queue_name)
+        if 'publish_time' not in extra:
+            extra['publish_time'] = round(time.time(), 4)
+        if 'publish_time_format':
+            extra['publish_time_format'] = time.strftime('%Y-%m-%d %H:%M:%S')
+
     def _submit_task(self, kw):
         while 1:  # 这一块的代码为支持暂停消费。
             # print(self._pause_flag)
@@ -396,7 +426,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             self._requeue(kw)
             time.sleep(self.time_interval_for_check_do_not_run_time)
             return
-
+        self._auto_fill_msg(kw['body'])
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
                 function_only_params):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。

{funboost-43.8 → funboost-44.0}/funboost/core/func_params_model.py

@@ -81,6 +81,14 @@ class BaseJsonAbleModel(BaseModel):
         # allow_mutation = False
         extra = "forbid"
 
+    @staticmethod
+    def init_by_another_model(model_type: typing.Type[BaseModel], modelx: BaseModel):
+        init_dict = {}
+        for k, v in modelx.dict().items():
+            if k in model_type.__fields__.keys():
+                init_dict[k] = v
+        return model_type(**init_dict)
+
 
 class FunctionResultStatusPersistanceConfig(BaseJsonAbleModel):
     is_save_status: bool  # 是否保存函数的运行状态信息
@@ -175,6 +183,8 @@ class BoosterParams(BaseJsonAbleModel):
     broker_exclusive_config: dict = {}  # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置，不是所有中间件都兼容的配置，因为框架支持30种消息队列，消息队列不仅仅是一般的先进先出queue这么简单的概念，
     # 例如kafka支持消费者组，rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制，有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义，可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
 
+    should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
+
     auto_generate_info: dict = {}  # 自动生成的信息,不需要用户主动传参.
 
     @root_validator(skip_on_failure=True)
@@ -194,7 +204,7 @@ class BoosterParams(BaseJsonAbleModel):
         # if not set(values.keys()).issubset(set(BoosterParams.__fields__.keys())):
         #     raise ValueError(f'{cls.__name__} 的字段包含了父类 BoosterParams 不存在的字段')
         for k in values.keys():
-            if k not in BoosterParams.model_fields.keys():
+            if k not in BoosterParams.__fields__.keys():
                 raise ValueError(f'{cls.__name__} 的字段新增了父类 BoosterParams 不存在的字段 "{k}"')  # 使 BoosterParams的子类,不能增加字段,只能覆盖字段.
         return values
 
@@ -255,6 +265,7 @@ class PublisherParams(BaseJsonAbleModel):
     consuming_function: typing.Callable = None  # consuming_function 作用是 inspect 模块获取函数的入参信息
     broker_kind: str = None
     broker_exclusive_config: dict = {}
+    should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
 
 
 if __name__ == '__main__':

{funboost-43.8 → funboost-44.0}/funboost/core/helper_funs.py

@@ -1,5 +1,6 @@
 import copy
 import time
+import uuid
 
 
 def get_publish_time(paramsx: dict):
@@ -42,4 +43,7 @@ def _try_get_user_funboost_common_config(funboost_common_conf_field:str):
         return getattr(funboost_config.FunboostCommonConfig,funboost_common_conf_field)
     except Exception as e:
         print(e)
-        return None
+        return None
+
+def generate_task_id(queue_name:str):
+    return f'{queue_name}_result:{uuid.uuid4()}'

{funboost-43.8 → funboost-44.0}/funboost/publishers/base_publisher.py

@@ -19,6 +19,7 @@ import amqpstorm
 
 import nb_log
 from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
+from funboost.core.helper_funs import generate_task_id
 from funboost.core.loggers import develop_logger
 
 from pikav1.exceptions import AMQPError as PikaAMQPError
@@ -188,9 +189,9 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         if 'extra' in msg:
             msg_function_kw.pop('extra')
             raw_extra = msg['extra']
-        if self.publish_params_checker:
+        if self.publish_params_checker and self.publisher_params.should_check_publish_func_params:
             self.publish_params_checker.check_params(msg_function_kw)
-        task_id = task_id or f'{self._queue_name}_result:{uuid.uuid4()}'
+        task_id = task_id or generate_task_id(self._queue_name)
         extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
                         'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
         if priority_control_config:

{funboost-43.8 → funboost-44.0/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.8
+Version: 44.0
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs