funboost 44.0__py3-none-any.whl → 44.2__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "44.0"
+__version__ = "44.2"
 
 from funboost.set_frame_config import show_frame_config
 
@@ -65,3 +65,4 @@ from funboost.core.current_task import funboost_current_task
 
 # 有的包默认没加handlers，原始的日志不漂亮且不可跳转不知道哪里发生的。这里把warnning级别以上的日志默认加上handlers。
 # nb_log.get_logger(name='', log_level_int=30, _log_filename='pywarning.log')
+

funboost/consumers/base_consumer.py

@@ -49,7 +49,7 @@ from funboost.concurrent_pool.single_thread_executor import SoloExecutor
 
 from funboost.core.function_result_status_saver import ResultPersistenceHelper, FunctionResultStatus, RunStatus
 
-from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, generate_task_id
+from funboost.core.helper_funs import delete_keys_and_return_new_dict, get_publish_time, MsgGenerater
 
 from funboost.concurrent_pool.async_helper import simple_run_in_executor
 from funboost.concurrent_pool.async_pool_executor import AsyncPoolExecutor
@@ -383,8 +383,11 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         raise NotImplementedError
 
-    def _auto_fill_msg(self, msg: dict):
-        """填充消息,消息没有使用funboost来发送,并且没有extra相关字段时候"""
+    def convert_msg_before_run(self, msg: dict):
+        """
+        转换消息,消息没有使用funboost来发送,并且没有extra相关字段时候
+        用户也可以按照4.21文档,继承任意Consumer类,并实现这个方法 convert_msg_before_run,先转换消息.
+        """
         """ 一般消息至少包含这样
         {
           "a": 42,
@@ -405,11 +408,11 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             msg['extra'] = {'is_auto_fill_extra': True}
         extra = msg['extra']
         if 'task_id' not in extra:
-            extra['task_id'] = generate_task_id(self._queue_name)
+            extra['task_id'] = MsgGenerater.generate_task_id(self._queue_name)
         if 'publish_time' not in extra:
-            extra['publish_time'] = round(time.time(), 4)
+            extra['publish_time'] = MsgGenerater.generate_publish_time()
         if 'publish_time_format':
-            extra['publish_time_format'] = time.strftime('%Y-%m-%d %H:%M:%S')
+            extra['publish_time_format'] = MsgGenerater.generate_publish_time_format()
 
     def _submit_task(self, kw):
         while 1:  # 这一块的代码为支持暂停消费。
@@ -426,7 +429,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             self._requeue(kw)
             time.sleep(self.time_interval_for_check_do_not_run_time)
             return
-        self._auto_fill_msg(kw['body'])
+        self.convert_msg_before_run(kw['body'])
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
                 function_only_params):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。
@@ -820,7 +823,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 self.logger.critical(msg=log_msg)
                 # noinspection PyProtectedMember,PyUnresolvedReferences
                 os._exit(444)
-            if self.consumer_params.function_timeout == 0:
+            if not self.consumer_params.function_timeout :
                 rs = await corotinue_obj
                 # rs = await asyncio.wait_for(corotinue_obj, timeout=4)
             else:

funboost/consumers/http_consumer.py

@@ -4,11 +4,11 @@
 import asyncio
 import json
 
-from aiohttp import web
-from aiohttp.web_request import Request
+# from aiohttp import web
+# from aiohttp.web_request import Request
 
 from funboost.consumers.base_consumer import AbstractConsumer
-
+from funboost.core.lazy_impoter import AioHttpImporter
 
 class HTTPConsumer(AbstractConsumer, ):
     """
@@ -38,26 +38,26 @@ class HTTPConsumer(AbstractConsumer, ):
         #
         # flask_app.run('0.0.0.0', port=self._port,debug=False)
 
-        routes = web.RouteTableDef()
+        routes = AioHttpImporter().web.RouteTableDef()
 
         # noinspection PyUnusedLocal
         @routes.get('/')
         async def hello(request):
-            return web.Response(text="Hello, from funboost")
+            return AioHttpImporter().web.Response(text="Hello, from funboost")
 
         @routes.post('/queue')
-        async def recv_msg(request: Request):
+        async def recv_msg(request: AioHttpImporter().Request):
             data = await request.post()
             msg = data['msg']
             kw = {'body': json.loads(msg)}
             self._submit_task(kw)
-            return web.Response(text="finish")
+            return AioHttpImporter().web.Response(text="finish")
 
-        app = web.Application()
+        app = AioHttpImporter().web.Application()
         app.add_routes(routes)
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
-        web.run_app(app, host='0.0.0.0', port=self._port, )
+        AioHttpImporter().web.run_app(app, host='0.0.0.0', port=self._port, )
 
     def _confirm_consume(self, kw):
         pass  # 没有确认消费的功能。

funboost/consumers/kafka_consumer.py

@@ -3,16 +3,14 @@
 # @Time    : 2022/8/8 0008 13:32
 import json
 # noinspection PyPackageRequirements
-from kafka import KafkaConsumer as OfficialKafkaConsumer, KafkaProducer, KafkaAdminClient
-# noinspection PyPackageRequirements
-from kafka.admin import NewTopic
-# noinspection PyPackageRequirements
-from kafka.errors import TopicAlreadyExistsError
+
 from funboost.constant import BrokerEnum
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import KafkaPythonImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
 # from nb_log import get_logger
 from funboost.core.loggers import get_funboost_file_logger
+
 # LogManager('kafka').get_logger_and_add_handlers(30)
 get_funboost_file_logger('kafka', log_level_int=30)
 
@@ -36,18 +34,18 @@ class KafkaConsumer(AbstractConsumer):
 
     def _shedual_task(self):
         try:
-            admin_client = KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client = KafkaPythonImporter().KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
 
-        self._producer = KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-        consumer = OfficialKafkaConsumer(self._queue_name, bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS,
-                                         group_id=self.consumer_params.broker_exclusive_config["group_id"],
-                                         enable_auto_commit=True,
-                                         auto_offset_reset=self.consumer_params.broker_exclusive_config["auto_offset_reset"],
-                                         )
+        self._producer = KafkaPythonImporter().KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+        consumer = KafkaPythonImporter().OfficialKafkaConsumer(self._queue_name, bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS,
+                                                               group_id=self.consumer_params.broker_exclusive_config["group_id"],
+                                                               enable_auto_commit=True,
+                                                               auto_offset_reset=self.consumer_params.broker_exclusive_config["auto_offset_reset"],
+                                                               )
         #  auto_offset_reset (str): A policy for resetting offsets on
         #             OffsetOutOfRange errors: 'earliest' will move to the oldest
         #             available message, 'latest' will move to the most recent. Any

funboost/consumers/kafka_consumer_manually_commit.py

@@ -10,13 +10,10 @@ from collections import defaultdict, OrderedDict
 import time
 
 # noinspection PyPackageRequirements
-from kafka import KafkaProducer, KafkaAdminClient
+# pip install kafka-python==2.0.2
 
-# noinspection PyPackageRequirements
-from kafka.admin import NewTopic
-# noinspection PyPackageRequirements
-from kafka.errors import TopicAlreadyExistsError
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import KafkaPythonImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
 from confluent_kafka.cimpl import TopicPartition
 from confluent_kafka import Consumer as ConfluentConsumer  # 这个包在win下不好安装，用户用这个中间件的时候自己再想办法安装。win用户需要安装c++ 14.0以上环境。
@@ -39,13 +36,13 @@ class KafkaConsumerManuallyCommit(AbstractConsumer):
     def _shedual_task(self):
 
         try:
-            admin_client = KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client = KafkaPythonImporter().KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
 
-        self._producer = KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+        self._producer = KafkaPythonImporter().KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
         # consumer 配置 https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
         self._confluent_consumer = ConfluentConsumer({
             'bootstrap.servers': ','.join(BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS),
@@ -130,14 +127,14 @@ class SaslPlainKafkaConsumer(KafkaConsumerManuallyCommit):
     def _shedual_task(self):
 
         try:
-            admin_client = KafkaAdminClient(
+            admin_client = KafkaPythonImporter().KafkaAdminClient(
                 **BrokerConnConfig.KFFKA_SASL_CONFIG)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
 
-        self._producer = KafkaProducer(
+        self._producer = KafkaPythonImporter().KafkaProducer(
             **BrokerConnConfig.KFFKA_SASL_CONFIG)
         # consumer 配置 https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md
         self._confluent_consumer = ConfluentConsumer({

funboost/consumers/mqtt_consumer.py

@@ -5,8 +5,9 @@ import json
 # import time
 from funboost.constant import BrokerEnum
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import PahoMqttImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
-import paho.mqtt.client as mqtt
+# import paho.mqtt.client as mqtt
 
 
 class MqttConsumer(AbstractConsumer):
@@ -23,7 +24,7 @@ class MqttConsumer(AbstractConsumer):
 
     # noinspection DuplicatedCode
     def _shedual_task(self):
-        client = mqtt.Client()
+        client = PahoMqttImporter().mqtt.Client()
         # client.username_pw_set('admin', password='public')
         client.on_connect = self._on_connect
         client.on_message = self._on_message

funboost/consumers/nats_consumer.py

@@ -1,7 +1,8 @@
 import json
-from pynats import NATSClient, NATSMessage  # noqa
-from funboost.constant import BrokerEnum
+# from pynats import NATSClient, NATSMessage  # noqa
+
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import NatsImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
 
 
@@ -13,10 +14,10 @@ class NatsConsumer(AbstractConsumer):
 
     def _shedual_task(self):
         # print(88888888888888)
-        nats_client = NATSClient(BrokerConnConfig.NATS_URL, socket_timeout=600, socket_keepalive=True)
+        nats_client = NatsImporter().NATSClient(BrokerConnConfig.NATS_URL, socket_timeout=600, socket_keepalive=True)
         nats_client.connect()
 
-        def callback(msg: NATSMessage):
+        def callback(msg: NatsImporter().NATSMessage):
             # print(type(msg))
             # print(msg.reply)
             # print(f"Received a message with subject {msg.subject}: {msg.payload}")

funboost/consumers/nsq_consumer.py

@@ -2,8 +2,10 @@
 # @Author  : ydf
 # @Time    : 2022/8/8 0008 13:32
 import json
-from gnsq import Consumer, Message
-from funboost.constant import BrokerEnum
+
+from funboost.core.lazy_impoter import GnsqImporter
+# from gnsq import Consumer, Message
+
 from funboost.funboost_config_deafult import BrokerConnConfig
 from funboost.consumers.base_consumer import AbstractConsumer
 # from nb_log import LogManager
@@ -19,11 +21,11 @@ class NsqConsumer(AbstractConsumer):
 
 
     def _shedual_task(self):
-        consumer = Consumer(self._queue_name, 'frame_channel', BrokerConnConfig.NSQD_TCP_ADDRESSES,
+        consumer = GnsqImporter().Consumer(self._queue_name, 'frame_channel', BrokerConnConfig.NSQD_TCP_ADDRESSES,
                             max_in_flight=self.consumer_params.concurrent_num, heartbeat_interval=60, timeout=600, )  # heartbeat_interval 不能设置为600
 
         @consumer.on_message.connect
-        def handler(consumerx: Consumer, message: Message):
+        def handler(consumerx: GnsqImporter().Consumer, message: GnsqImporter().Message):
             # 第一条消息不能并发，第一条消息之后可以并发。
             self._print_message_get_from_broker('nsq', message.body.decode())
             # self.logger.debug(f'从nsq的 [{self._queue_name}] 主题中 取出的消息是：  {message.body.decode()}')

funboost/consumers/zeromq_consumer.py

@@ -4,10 +4,11 @@ import os
 import socket
 import json
 # import time
-import zmq
+# import zmq
 import multiprocessing
 from funboost.constant import BrokerEnum
 from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.core.lazy_impoter import ZmqImporter
 # from nb_log import get_logger
 from funboost.core.loggers import get_funboost_file_logger
 
@@ -32,17 +33,17 @@ logger_zeromq_broker = get_funboost_file_logger('zeromq_broker')
 # noinspection PyUnresolvedReferences
 def start_broker(port_router: int, port_dealer: int):
     try:
-        context = zmq.Context()
+        context = ZmqImporter().zmq.Context()
         # noinspection PyUnresolvedReferences
-        frontend = context.socket(zmq.ROUTER)
-        backend = context.socket(zmq.DEALER)
+        frontend = context.socket(ZmqImporter().zmq.ROUTER)
+        backend = context.socket(ZmqImporter().zmq.DEALER)
         frontend.bind(f"tcp://*:{port_router}")
         backend.bind(f"tcp://*:{port_dealer}")
 
         # Initialize poll set
-        poller = zmq.Poller()
-        poller.register(frontend, zmq.POLLIN)
-        poller.register(backend, zmq.POLLIN)
+        poller = ZmqImporter().zmq.Poller()
+        poller.register(frontend, ZmqImporter().zmq.POLLIN)
+        poller.register(backend, ZmqImporter().zmq.POLLIN)
         logger_zeromq_broker.info(f'broker 绑定端口  {port_router}   {port_dealer}  成功')
 
         # Switch messages between sockets
@@ -50,11 +51,11 @@ def start_broker(port_router: int, port_dealer: int):
         while True:
             socks = dict(poller.poll())  # 轮询器 循环接收
 
-            if socks.get(frontend) == zmq.POLLIN:
+            if socks.get(frontend) == ZmqImporter().zmq.POLLIN:
                 message = frontend.recv_multipart()
                 backend.send_multipart(message)
 
-            if socks.get(backend) == zmq.POLLIN:
+            if socks.get(backend) == ZmqImporter().zmq.POLLIN:
                 message = backend.recv_multipart()
                 frontend.send_multipart(message)
     except BaseException as e:
@@ -87,9 +88,9 @@ class ZeroMqConsumer(AbstractConsumer):
     # noinspection DuplicatedCode
     def _shedual_task(self):
         self.start_broker_queue_name_as_port()
-        context = zmq.Context()
+        context = ZmqImporter().zmq.Context()
         # noinspection PyUnresolvedReferences
-        zsocket = context.socket(zmq.REP)
+        zsocket = context.socket(ZmqImporter().zmq.REP)
         zsocket.connect(f"tcp://localhost:{int(self._queue_name) + 1}")
 
         while True:

funboost/core/fabric_deploy_helper.py

@@ -103,7 +103,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
         process_mark = f'funboost_fabric_mark__{queue_name}__{func_name}'
         conn = Connection(host, port=port, user=user, connect_kwargs={"password": password}, )
         kill_shell = f'''ps -aux|grep {process_mark}|grep -v grep|awk '{{print $2}}' |xargs kill -9'''
-        logger.warning(f'{kill_shell} 命令杀死 {process_mark} 标识的进程')
+        logger.warning(f'使用linux命令 {kill_shell} 命令杀死 {process_mark} 标识的进程')
         # uploader.ssh.exec_command(kill_shell)
         conn.run(kill_shell, encoding='utf-8', warn=True)  # 不想提示，免得烦扰用户以为有什么异常了。所以用上面的paramiko包的ssh.exec_command
 
@@ -113,7 +113,7 @@ def fabric_deploy(booster: Booster, host, port, user, password,
         if not extra_shell_str2.endswith(';') and extra_shell_str != '':
             extra_shell_str2 += ';'
         shell_str = extra_shell_str2 + shell_str
-        logger.warning(f'使用语句 {shell_str} 在远程机器 {host} 上启动任务消费')
+        logger.warning(f'使用linux命令 {shell_str} 在远程机器 {host} 上启动任务消费')
         conn.run(shell_str, encoding='utf-8', **invoke_runner_kwargs)
         # uploader.ssh.exec_command(shell_str)
 
@@ -125,6 +125,6 @@ def kill_all_remote_tasks(host, port, user, password):
     uploader = ParamikoFolderUploader(host, port, user, password, '', '')
     funboost_fabric_mark_all = 'funboost_fabric_mark__'
     kill_shell = f'''ps -aux|grep {funboost_fabric_mark_all}|grep -v grep|awk '{{print $2}}' |xargs kill -9'''
-    logger.warning(f'{kill_shell} 命令杀死 {funboost_fabric_mark_all} 标识的进程')
+    logger.warning(f'使用linux命令 {kill_shell} 命令杀死 {funboost_fabric_mark_all} 标识的进程')
     uploader.ssh.exec_command(kill_shell)
     logger.warning(f'杀死 {host}  机器所有的 {funboost_fabric_mark_all} 标识的进程')

funboost/core/funboost_config_getter.py

@@ -0,0 +1,7 @@
+def _try_get_user_funboost_common_config(funboost_common_conf_field:str):
+    try:
+        import funboost_config  # 第一次启动funboost前还没这个文件,或者还没有初始化配置之前,就要使用使用配置.
+        return getattr(funboost_config.FunboostCommonConfig,funboost_common_conf_field)
+    except Exception as e:
+        print(e)
+        return None

funboost/core/funboost_time.py

@@ -0,0 +1,28 @@
+import pytz
+import time
+
+import datetime
+
+import typing
+
+from nb_time import NbTime
+from funboost.funboost_config_deafult import FunboostCommonConfig
+
+class FunboostTime(NbTime):
+    default_formatter = NbTime.FORMATTER_DATETIME_NO_ZONE
+    def get_time_zone_str(self,time_zone: typing.Union[str, datetime.tzinfo,None] = None):
+        return time_zone or self.default_time_zone or  FunboostCommonConfig.TIMEZONE  or self.get_localzone_name()
+
+
+
+if __name__ == '__main__':
+    print(NbTime())
+    for i in range(100000):
+        # print(generate_publish_time())
+        # print(generate_publish_time_format())
+        # generate_publish_time()
+        # generate_publish_time_format()
+
+        datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).strftime(NbTime.FORMATTER_DATETIME_NO_ZONE)
+        datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).timestamp()
+    print(NbTime())

funboost/core/func_params_model.py

@@ -123,7 +123,7 @@ class BoosterParams(BaseJsonAbleModel):
     由于有很好用的qps控制运行频率和智能扩大缩小的线程池，此框架建议不需要理会和设置并发数量只需要关心qps就行了，框架的并发是自适应并发数量，这一点很强很好用。"""
     concurrent_mode: str = ConcurrentModeEnum.THREADING  # 并发模式,支持THREADING,GEVENT,EVENTLET,ASYNC,SINGLE_THREAD并发,multi_process_consume 支持协程/线程 叠加多进程并发,性能炸裂.
     concurrent_num: int = 50  # 并发数量，并发种类由concurrent_mode决定
-    specify_concurrent_pool: FunboostBaseConcurrentPool = None  # 使用指定的线程池/携程池，可以多个消费者共使用一个线程池,节约线程.不为None时候。threads_num失效
+    specify_concurrent_pool: typing.Optional[FunboostBaseConcurrentPool] = None  # 使用指定的线程池/携程池，可以多个消费者共使用一个线程池,节约线程.不为None时候。threads_num失效
     specify_async_loop: asyncio.AbstractEventLoop = None  # 指定的async的loop循环，设置并发模式为async才能起作用。 有些包例如aiohttp,请求和httpclient的实例化不能处在两个不同的loop中,可以传过来.
 
     """qps:
@@ -159,7 +159,7 @@ class BoosterParams(BaseJsonAbleModel):
     msg_expire_senconds: typing.Union[float, int] = None  # 消息过期时间,可以设置消息是多久之前发布的就丢弃这条消息,不运行. 为None则永不丢弃
 
     do_task_filtering: bool = False  # 是否对函数入参进行过滤去重.
-    task_filtering_expire_seconds: int = 0  # 任务过滤的失效期，为0则永久性过滤任务。例如设置过滤过期时间是1800秒 ， 30分钟前发布过1 + 2 的任务，现在仍然执行，如果是30分钟以内发布过这个任务，则不执行1 + 2
+    task_filtering_expire_seconds: int = 0  # 任务过滤的失效期，为0则永久性过滤任务。例如设置过滤过期时间是1800秒 ， 30分钟前发布过1 + 2 的任务，现在仍然执行，如果是30分钟以内执行过这个任务，则不执行1 + 2
 
     function_result_status_persistance_conf: FunctionResultStatusPersistanceConfig = FunctionResultStatusPersistanceConfig(
         is_save_result=False, is_save_status=False, expire_seconds=7 * 24 * 3600, is_use_bulk_insert=False)  # 是否保存函数的入参，运行结果和运行状态到mongodb。这一步用于后续的参数追溯，任务统计和web展示，需要安装mongo。
@@ -183,7 +183,7 @@ class BoosterParams(BaseJsonAbleModel):
     broker_exclusive_config: dict = {}  # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置，不是所有中间件都兼容的配置，因为框架支持30种消息队列，消息队列不仅仅是一般的先进先出queue这么简单的概念，
     # 例如kafka支持消费者组，rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制，有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义，可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
 
-    should_check_publish_func_params: bool = True # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
+    should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
 
     auto_generate_info: dict = {}  # 自动生成的信息,不需要用户主动传参.
 
@@ -277,3 +277,4 @@ if __name__ == '__main__':
     # print(PriorityConsumingControlConfig().get_str_dict())
 
     print(BoosterParams(queue_name='3213', specify_concurrent_pool=FlexibleThreadPool(100)).json_pre())
+    print(PublisherParams.schema_json())

funboost/core/helper_funs.py

@@ -2,6 +2,8 @@ import copy
 import time
 import uuid
 
+from funboost.core.funboost_time import FunboostTime
+
 
 def get_publish_time(paramsx: dict):
     """
@@ -37,13 +39,23 @@ def block_python_main_thread_exit():
 run_forever = block_python_main_thread_exit
 
 
-def _try_get_user_funboost_common_config(funboost_common_conf_field:str):
-    try:
-        import funboost_config  # 第一次启动funboost前还没这个文件,或者还没有初始化配置之前,就要使用使用配置.
-        return getattr(funboost_config.FunboostCommonConfig,funboost_common_conf_field)
-    except Exception as e:
-        print(e)
-        return None
+class MsgGenerater:
+    @staticmethod
+    def generate_task_id(queue_name:str) -> str:
+        return f'{queue_name}_result:{uuid.uuid4()}'
+
+    @staticmethod
+    def generate_publish_time() -> float:
+        return round(FunboostTime().timestamp,4)
+
+    @staticmethod
+    def generate_publish_time_format() -> str:
+        return FunboostTime().get_str()
+
+    @classmethod
+    def generate_pulish_time_and_task_id(cls,queue_name:str,task_id=None):
+        extra_params = {'task_id': task_id or cls.generate_task_id(queue_name), 'publish_time': cls.generate_publish_time(),
+                        'publish_time_format': cls.generate_publish_time_format()}
+        return extra_params
+
 
-def generate_task_id(queue_name:str):
-    return f'{queue_name}_result:{uuid.uuid4()}'

funboost/core/lazy_impoter.py

@@ -2,8 +2,11 @@ import abc
 
 from funboost.utils.decorators import cached_method_result, singleton, SingletonBaseNew, SingletonBaseCustomInit
 
+"""
+延迟导入
+或者使用时候再pip安装
+"""
 
-# @singleton   # 不方便代码补全
 
 class LazyImpoter(SingletonBaseNew):
     """
@@ -79,20 +82,99 @@ class EventletImporter:
         self.patcher = patcher
         self.Timeout = Timeout
 
+
 @singleton
 class PeeweeImporter:
     def __init__(self):
-        '''pip install peewee == 3.17'''
+        """pip install peewee == 3.17"""
         from peewee import ModelSelect, Model, BigAutoField, CharField, DateTimeField, MySQLDatabase
         from playhouse.shortcuts import model_to_dict, dict_to_model
         self.ModelSelect = ModelSelect
         self.Model = Model
         self.BigAutoField = BigAutoField
         self.CharField = CharField
-        self.DateTimeField =DateTimeField
+        self.DateTimeField = DateTimeField
         self.MySQLDatabase = MySQLDatabase
         self.model_to_dict = model_to_dict
-        self.dict_to_model =dict_to_model
+        self.dict_to_model = dict_to_model
+
+
+@singleton
+class AioHttpImporter:
+
+    def __init__(self):
+        """pip install aiohttp==3.8.3"""
+        from aiohttp import web
+        from aiohttp.web_request import Request
+        self.web = web
+        self.Request = Request
+
+
+@singleton
+class NatsImporter:
+    def __init__(self):
+        """pip install nats-python """
+        from pynats import NATSClient, NATSMessage
+        self.NATSClient = NATSClient
+        self.NATSMessage = NATSMessage
+
+@singleton
+class GnsqImporter:
+    def __init__(self):
+        """pip install  gnsq==1.0.1"""
+        from gnsq import Consumer, Message
+        from gnsq import Producer, NsqdHTTPClient
+        from gnsq.errors import NSQHttpError
+        self.Consumer = Consumer
+        self.Message = Message
+        self.Producer = Producer
+        self.NsqdHTTPClient = NsqdHTTPClient
+        self.NSQHttpError = NSQHttpError
+
+@singleton
+class ElasticsearchImporter:
+    def __init__(self):
+        """pip install elasticsearch """
+        from elasticsearch import helpers
+        self.helpers = helpers
+
+
+@singleton
+class PsutilImporter:
+    def __init__(self):
+        """pip install  psutil"""
+        import psutil
+        self.psutil = psutil
+
+@singleton
+class PahoMqttImporter:
+    def __init__(self):
+        """pip install paho-mqtt"""
+        import paho.mqtt.client as mqtt
+        self.mqtt = mqtt
+
+@singleton
+class ZmqImporter:
+    def __init__(self):
+        """pip install zmq pyzmq"""
+        import zmq
+        self.zmq = zmq
+
+@singleton
+class KafkaPythonImporter:
+    def __init__(self):
+        """pip install kafka-python==2.0.2"""
+
+        from kafka import KafkaConsumer as OfficialKafkaConsumer, KafkaProducer, KafkaAdminClient
+        from kafka.admin import NewTopic
+        from kafka.errors import TopicAlreadyExistsError
+
+        self.OfficialKafkaConsumer = OfficialKafkaConsumer
+        self.KafkaProducer = KafkaProducer
+        self.KafkaAdminClient = KafkaAdminClient
+        self.NewTopic = NewTopic
+        self.TopicAlreadyExistsError = TopicAlreadyExistsError
+
 
 if __name__ == '__main__':
     for i in range(10000):

funboost/core/loggers.py

@@ -1,5 +1,5 @@
 import nb_log
-from funboost.core.helper_funs import _try_get_user_funboost_common_config
+from funboost.core.funboost_config_getter import _try_get_user_funboost_common_config
 
 # noinspection PyUnresolvedReferences
 from nb_log import get_logger, LoggerLevelSetterMixin, nb_log_config_default

funboost/function_result_web/app.py

@@ -5,6 +5,9 @@
 import datetime
 import json
 
+"""
+pip install Flask flask_bootstrap  flask_wtf  wtforms flask_login       
+"""
 from flask import render_template, Flask, request, url_for, jsonify, flash, redirect
 from flask_bootstrap import Bootstrap
 from flask_wtf import FlaskForm

funboost/function_result_web/functions.py

@@ -5,7 +5,6 @@ import datetime
 import json
 from pprint import pprint
 import time
-from flask import jsonify
 import copy
 from funboost import nb_print
 from funboost.utils import time_util, decorators, LoggerMixin