funboost 43.8__py3-none-any.whl → 43.9__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "43.8"
+__version__ = "43.9"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/consumers/base_consumer.py

@@ -33,7 +33,7 @@ import nb_log
 from funboost.core.current_task import funboost_current_task, get_current_taskid
 from funboost.core.loggers import develop_logger
 
-from funboost.core.func_params_model import BoosterParams, PublisherParams
+from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
 from funboost.core.task_id_logger import TaskIdLogger
 from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, CompatibleLogger,
                     LoggerMixinDefaultWithFileHandler, stdout_write, is_main_process,
@@ -49,7 +49,7 @@ from funboost.concurrent_pool.single_thread_executor import SoloExecutor
 
 from funboost.core.function_result_status_saver import ResultPersistenceHelper, FunctionResultStatus, RunStatus
 
-from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time
+from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, generate_task_id
 
 from funboost.concurrent_pool.async_helper import simple_run_in_executor
 from funboost.concurrent_pool.async_pool_executor import AsyncPoolExecutor
@@ -215,13 +215,15 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._consuming_function_is_asyncio = inspect.iscoroutinefunction(self.consuming_function)
         self.custom_init()
         # develop_logger.warning(consumer_params._log_filename)
-        self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
-                                                broker_kind=self.BROKER_KIND, log_level=consumer_params.log_level,
-                                                logger_prefix=consumer_params.logger_prefix,
-                                                create_logger_file=consumer_params.create_logger_file,
-                                                log_filename=consumer_params.log_filename,
-                                                logger_name=consumer_params.logger_name,
-                                                broker_exclusive_config=self.consumer_params.broker_exclusive_config)
+        # self.publisher_params = PublisherParams(queue_name=consumer_params.queue_name, consuming_function=consumer_params.consuming_function,
+        #                                         broker_kind=self.BROKER_KIND, log_level=consumer_params.log_level,
+        #                                         logger_prefix=consumer_params.logger_prefix,
+        #                                         create_logger_file=consumer_params.create_logger_file,
+        #                                         log_filename=consumer_params.log_filename,
+        #                                         logger_name=consumer_params.logger_name,
+        #                                         broker_exclusive_config=self.consumer_params.broker_exclusive_config)
+        self.publisher_params = BaseJsonAbleModel.init_by_another_model(PublisherParams, consumer_params)
+        # print(self.publisher_params)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
         atexit.register(self.join_shedual_task_thread)
@@ -381,6 +383,33 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         raise NotImplementedError
 
+    def __auto_fill_msg(self, msg: dict):
+        """填充消息,消息没有使用funboost来发送,并且没有extra相关字段时候"""
+        """ 一般消息至少包含这样
+       {
+  "a": 42,
+  "b": 84,
+  "extra": {
+    "task_id": "queue_2_result:9b79a372-f765-4a33-8639-9d15d7a95f61",
+    "publish_time": 1701687443.3596,
+    "publish_time_format": "2023-12-04 18:57:23"
+  }
+}
+        """
+        """
+        extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
+                        'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
+        """
+        if 'extra' not in msg:
+            msg['extra'] = {}
+        extra = msg['extra']
+        if 'task_id' not in extra:
+            extra['task_id'] = generate_task_id(self._queue_name)
+        if 'publish_time' not in extra:
+            extra['publish_time'] = round(time.time(), 4)
+        if 'publish_time_format':
+            extra['publish_time_format'] = time.strftime('%Y-%m-%d %H:%M:%S')
+
     def _submit_task(self, kw):
         while 1:  # 这一块的代码为支持暂停消费。
             # print(self._pause_flag)
@@ -396,7 +425,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             self._requeue(kw)
             time.sleep(self.time_interval_for_check_do_not_run_time)
             return
-
+        self.__auto_fill_msg(kw['body'])
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
                 function_only_params):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。

funboost/core/func_params_model.py

@@ -81,6 +81,14 @@ class BaseJsonAbleModel(BaseModel):
         # allow_mutation = False
         extra = "forbid"
 
+    @staticmethod
+    def init_by_another_model(model_type: typing.Type[BaseModel], modelx: BaseModel):
+        init_dict = {}
+        for k, v in modelx.dict().items():
+            if k in model_type.__fields__.keys():
+                init_dict[k] = v
+        return model_type(**init_dict)
+
 
 class FunctionResultStatusPersistanceConfig(BaseJsonAbleModel):
     is_save_status: bool  # 是否保存函数的运行状态信息
@@ -175,6 +183,8 @@ class BoosterParams(BaseJsonAbleModel):
     broker_exclusive_config: dict = {}  # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置，不是所有中间件都兼容的配置，因为框架支持30种消息队列，消息队列不仅仅是一般的先进先出queue这么简单的概念，
     # 例如kafka支持消费者组，rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制，有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义，可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
 
+    should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
+
     auto_generate_info: dict = {}  # 自动生成的信息,不需要用户主动传参.
 
     @root_validator(skip_on_failure=True)
@@ -194,7 +204,7 @@ class BoosterParams(BaseJsonAbleModel):
         # if not set(values.keys()).issubset(set(BoosterParams.__fields__.keys())):
         #     raise ValueError(f'{cls.__name__} 的字段包含了父类 BoosterParams 不存在的字段')
         for k in values.keys():
-            if k not in BoosterParams.model_fields.keys():
+            if k not in BoosterParams.__fields__.keys():
                 raise ValueError(f'{cls.__name__} 的字段新增了父类 BoosterParams 不存在的字段 "{k}"')  # 使 BoosterParams的子类,不能增加字段,只能覆盖字段.
         return values
 
@@ -255,6 +265,7 @@ class PublisherParams(BaseJsonAbleModel):
     consuming_function: typing.Callable = None  # consuming_function 作用是 inspect 模块获取函数的入参信息
     broker_kind: str = None
     broker_exclusive_config: dict = {}
+    should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
 
 
 if __name__ == '__main__':

funboost/core/helper_funs.py

@@ -1,5 +1,6 @@
 import copy
 import time
+import uuid
 
 
 def get_publish_time(paramsx: dict):
@@ -42,4 +43,7 @@ def _try_get_user_funboost_common_config(funboost_common_conf_field:str):
         return getattr(funboost_config.FunboostCommonConfig,funboost_common_conf_field)
     except Exception as e:
         print(e)
-        return None
+        return None
+
+def generate_task_id(queue_name:str):
+    return f'{queue_name}_result:{uuid.uuid4()}'

funboost/publishers/base_publisher.py

@@ -19,6 +19,7 @@ import amqpstorm
 
 import nb_log
 from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
+from funboost.core.helper_funs import generate_task_id
 from funboost.core.loggers import develop_logger
 
 from pikav1.exceptions import AMQPError as PikaAMQPError
@@ -188,9 +189,9 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         if 'extra' in msg:
             msg_function_kw.pop('extra')
             raw_extra = msg['extra']
-        if self.publish_params_checker:
+        if self.publish_params_checker and self.publisher_params.should_check_publish_func_params:
             self.publish_params_checker.check_params(msg_function_kw)
-        task_id = task_id or f'{self._queue_name}_result:{uuid.uuid4()}'
+        task_id = task_id or generate_task_id(self._queue_name)
         extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
                         'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
         if priority_control_config:

{funboost-43.8.dist-info → funboost-43.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.8
+Version: 43.9
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.8.dist-info → funboost-43.9.dist-info}/RECORD

@@ -1,4 +1,4 @@
-funboost/__init__.py,sha256=wESWldi73wbFUQRKyLjd4WcKNod-qHTxLtJNXoOnIII,3832
+funboost/__init__.py,sha256=H-RYbw9IE6_YoCnuABJoytt_IAGKkpt-IFzEY8GtSHo,3832
 funboost/__init__old.py,sha256=07A1MLsxLtuRQQOIfDyphddOwNBrGe34enoHWAnjV14,20379
 funboost/__main__.py,sha256=-6Nogi666Y0LN8fVm3JmHGTOk8xEGWvotW_GDbSaZME,1065
 funboost/constant.py,sha256=Yxt3WJt9D8ybcrgiojOy0qjnq5mffwTnTJplerGL0Oo,7188
@@ -33,7 +33,7 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=RVUZiylUvpTm6U
 funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=KL6zEQaa1KkZOlAO85mCC1gwLm-YC5Ghn21IUom0UKM,9598
 funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=OHMWJ9l3EYTpPpcrPrGGKd4K0tmQ2PN8HiX0Dta0EOo,5728
 funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
-funboost/consumers/base_consumer.py,sha256=5e-nHq6lGAnYzP0zLkbvtRFJPhHhZawTiiv1sJfYisg,74479
+funboost/consumers/base_consumer.py,sha256=rUxBGuvadSBxNRw2uoZpC9plMUxwQBWgjQqAf4Wo1mU,75738
 funboost/consumers/celery_consumer.py,sha256=9gtz7nlZkmv3ErmaseT0_Q__ltSPx-fOcwi-TMPoaLA,9220
 funboost/consumers/confirm_mixin.py,sha256=eY6fNwx51Hn4bQSYRjyTRwOqfCGsikVnd2Ga_Ep31N4,6062
 funboost/consumers/dramatiq_consumer.py,sha256=ozmeAfeF0U-YNYHK4suQB0N264h5AZdfMH0O45Mh-8A,2229
@@ -85,10 +85,10 @@ funboost/core/booster.py,sha256=BMSrQbpvIfhigzHjloFwHb7mlw69bvk48RJN43dDiTQ,1591
 funboost/core/current_task.py,sha256=rAIQVLqYqtlHVRkjl17yki-mqvuMb640ssGmto4RSdA,4864
 funboost/core/exceptions.py,sha256=pLF7BkRJAfDiWp2_xGZqencmwdPiSQI1NENbImExknY,1311
 funboost/core/fabric_deploy_helper.py,sha256=AbiQqCCDCnCf9vU4rhaAGLRiS4XBe5_WMnoHmAysYto,9958
-funboost/core/func_params_model.py,sha256=ChxvSq_iVuYnGR_CdN9h9qNtvBq8sZ4BocdzCjgsQHc,18167
+funboost/core/func_params_model.py,sha256=9LbfO0HnRmTph0qy8BDvBtoYurGXPfnX6hAiUzPM4ZI,19115
 funboost/core/function_result_status_config.py,sha256=PyjqAQOiwsLt28sRtH-eYRjiI3edPFO4Nde0ILFRReE,1764
 funboost/core/function_result_status_saver.py,sha256=UdokGSwU630t70AZnT9Ecj7GpYXORBDivlc9kadoI2E,9172
-funboost/core/helper_funs.py,sha256=M9Ad9EzgHdP581X-vuFgCavJRoezLGXlXSFg7zyMWD0,1578
+funboost/core/helper_funs.py,sha256=k5HttGoz-E9doLBFz0P0b8SCiCWXuk3ZYD_A-tAfn0Q,1682
 funboost/core/kill_remote_task.py,sha256=MZ5vWLGt6SxyN76h5Lf_id9tyVUzjR-qXNyJwXaGlZY,8129
 funboost/core/lazy_impoter.py,sha256=k0qZs39VL5qb7FspMt0x1noxqh3RIzt_czlFBwcyvKA,2722
 funboost/core/loggers.py,sha256=173aXdqE8nAe8t6OcVMNAFsCUClVrWQovdQhTAg9IyM,2407
@@ -122,7 +122,7 @@ funboost/function_result_web/static/js/jquery-1.11.0.min.js,sha256=ryQZ3RXgnqkTz
 funboost/function_result_web/templates/index.html,sha256=YM0582Q4t2da-xBf3Ga0McIfcsT9H98rjZck-irMkGo,20387
 funboost/function_result_web/templates/login.html,sha256=q37dj7O0LeyiV38Zd5P1Qn_qmhjdFomuYTRY1Yk48Bo,2007
 funboost/publishers/__init__.py,sha256=xqBHlvsJQVPfbdvP84G0LHmVB7-pFBS7vDnX1Uo9pVY,131
-funboost/publishers/base_publisher.py,sha256=q78zraFleZv4YkVABFW1WEwMIgZ0TKozJZkp6RbwHXc,15070
+funboost/publishers/base_publisher.py,sha256=81Lj8atdD9_Hyh4cGJCa84zhxKQzqV13x2bax58fQwc,15176
 funboost/publishers/celery_publisher.py,sha256=uc9N1uLW74skUCw8dsnvxORM2O3cy4SiI7tUZRmvkHA,2336
 funboost/publishers/celery_publisher000.py,sha256=2XLOyU2__vlIUTi5L15uf0BJqAIjxbc3kCLIRDSOY9w,3966
 funboost/publishers/confluent_kafka_publisher.py,sha256=gC4SUk6I_7zjSngcbTI7oTJ7sza3oE3PE19KQkwCpn4,4802
@@ -264,9 +264,9 @@ funboost/utils/pysnooper_ydf/utils.py,sha256=evSmGi_Oul7vSP47AJ0DLjFwoCYCfunJZ1m
 funboost/utils/pysnooper_ydf/variables.py,sha256=QejRDESBA06KG9OH4sBT4J1M55eaU29EIHg8K_igaXo,3693
 funboost/utils/times/__init__.py,sha256=Y4bQD3SIA_E7W2YvHq2Qdi0dGM4H2DxyFNdDOuFOq1w,2417
 funboost/utils/times/version.py,sha256=11XfnZVVzOgIhXXdeN_mYfdXThfrsbQHpA0wCjz-hpg,17
-funboost-43.8.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
-funboost-43.8.dist-info/METADATA,sha256=a6DT6uVuO6B454V1el0hNEW69IZU4kSh52Qow-3qrWM,30408
-funboost-43.8.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
-funboost-43.8.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
-funboost-43.8.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
-funboost-43.8.dist-info/RECORD,,
+funboost-43.9.dist-info/LICENSE,sha256=9EPP2ktG_lAPB8PjmWV-c9BiaJHc_FP6pPLcUrUwx0E,11562
+funboost-43.9.dist-info/METADATA,sha256=7fO2oWHqPhj3WOk1X00OvefH5GGGx8Rsx2n7wYPEzWI,30408
+funboost-43.9.dist-info/WHEEL,sha256=yQN5g4mg4AybRjkgi-9yy4iQEFibGQmlz78Pik5Or-A,92
+funboost-43.9.dist-info/entry_points.txt,sha256=yMSSAGRzRAAhGyNNQHw24MooKlDZsaJ499_D6fPl58A,96
+funboost-43.9.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
+funboost-43.9.dist-info/RECORD,,