funboost 44.3__py3-none-any.whl → 44.5__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "44.3"
+__version__ = "44.5"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/concurrent_pool/async_helper.py

@@ -2,9 +2,11 @@ from functools import partial
 import asyncio
 from concurrent.futures import Executor
 from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
+# from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
 
 # 没有使用内置的concurrent.futures.ThreadpoolExecutor线程池，而是使用智能伸缩线程池。
-async_executor_default = ThreadPoolExecutorShrinkAble()
+async_executor_default = ThreadPoolExecutorShrinkAble(500)
+# async_executor_default = FlexibleThreadPool(50)  # 这个不支持future特性
 
 
 async def simple_run_in_executor(f, *args, async_executor: Executor = None, async_loop=None, **kwargs):

funboost/constant.py

@@ -3,14 +3,20 @@ class BrokerEnum:
     RABBITMQ_AMQPSTORM = 'RABBITMQ_AMQPSTORM'  # 使用 amqpstorm 包操作rabbitmq  作为 分布式消息队列，支持消费确认.强烈推荐这个作为funboost中间件。
     RABBITMQ = RABBITMQ_AMQPSTORM
 
-    RABBITMQ_RABBITPY = 'RABBITMQ_RABBITPY'  # 使用 rabbitpy 包操作rabbitmq  作为 分布式消息队列，支持消费确认。
+    RABBITMQ_RABBITPY = 'RABBITMQ_RABBITPY'  # 使用 rabbitpy 包操作rabbitmq  作为 分布式消息队列，支持消费确认，不建议使用
 
     REDIS = 'REDIS'  # 使用 redis 的 list结构，brpop 作为分布式消息队列。随意重启和关闭会丢失大量消息，不支持消费确认。注重性能不在乎丢失消息可以选这个redis方案。
+    REDIS_ACK_ABLE = 'REDIS_ACK_ABLE'  # 基于redis的 list + 临时unack的set队列，采用了 lua脚本操持了取任务和加到pengding为原子性，,基于进程心跳消失判断消息是否为掉线进程的，随意重启和掉线不会丢失任务。
+    REIDS_ACK_USING_TIMEOUT = 'reids_ack_using_timeout'  # 基于redis的 list + 临时unack的set队列，使用超时多少秒没确认消费就自动重回队列，请注意 ack_timeout的设置值和函数耗时大小，否则会发生反复重回队列的后果,boost可以设置ack超时，broker_exclusive_config={'ack_timeout': 1800}
+    REDIS_PRIORITY = 'REDIS_PRIORITY'  # # 基于redis的多 list + 临时unack的set队列，blpop监听多个key，和rabbitmq的x-max-priority属性一样，支持任务优先级。看文档4.29优先级队列说明。
+    REDIS_STREAM = 'REDIS_STREAM'  # 基于redis 5.0 版本以后，使用 stream 数据结构作为分布式消息队列，支持消费确认和持久化和分组消费，是redis官方推荐的消息队列形式，比list结构更适合。
+    RedisBrpopLpush = 'RedisBrpopLpush'  # 基于redis的list结构但是采用brpoplpush 双队列形式，和 redis_ack_able的实现差不多，实现上采用了原生命令就不需要lua脚本来实现取出和加入unack了。
+    REDIS_PUBSUB = 'REDIS_PUBSUB'  # 基于redis 发布订阅的，发布一个消息多个消费者都能收到同一条消息，但不支持持久化
 
     MEMORY_QUEUE = 'MEMORY_QUEUE'  # 使用python queue.Queue实现的基于当前python进程的消息队列，不支持跨进程 跨脚本 跨机器共享任务，不支持持久化，适合一次性短期简单任务。
     LOCAL_PYTHON_QUEUE = MEMORY_QUEUE  # 别名，python本地queue就是基于python自带的语言的queue.Queue，消息存在python程序的内存中，不支持重启断点接续。
 
-    RABBITMQ_PIKA = 'RABBITMQ_PIKA'  # 使用pika包操作rabbitmq  作为 分布式消息队列。
+    RABBITMQ_PIKA = 'RABBITMQ_PIKA'  # 使用pika包操作rabbitmq  作为 分布式消息队列。，不建议使用
 
     MONGOMQ = 'MONGOMQ'  # 使用mongo的表中的行模拟的 作为分布式消息队列，支持消费确认。
 
@@ -27,19 +33,13 @@ class BrokerEnum:
 
     KAFKA_CONFLUENT_SASlPlAIN = 'KAFKA_CONFLUENT_SASlPlAIN'  # 可以设置账号密码的kafka
 
-    REDIS_ACK_ABLE = 'REDIS_ACK_ABLE'  # 基于redis的 list + 临时unack的set队列，采用了 lua脚本操持了取任务和加到pengding为原子性，随意重启和掉线不会丢失任务。
-
-    REDIS_PRIORITY = 'REDIS_PRIORITY'  # # 基于redis的多 list + 临时unack的set队列，blpop监听多个key，和rabbitmq的x-max-priority属性一样，支持任务优先级。看文档4.29优先级队列说明。
-
     SQLACHEMY = 'SQLACHEMY'  # 基于SQLACHEMY 的连接作为分布式消息队列中间件支持持久化和消费确认。支持mysql oracle sqlserver等5种数据库。
 
     ROCKETMQ = 'ROCKETMQ'  # 基于 rocketmq 作为分布式消息队列，这个中间件必须在linux下运行，win不支持。
 
-    REDIS_STREAM = 'REDIS_STREAM'  # 基于redis 5.0 版本以后，使用 stream 数据结构作为分布式消息队列，支持消费确认和持久化和分组消费，是redis官方推荐的消息队列形式，比list结构更适合。
-
     ZEROMQ = 'ZEROMQ'  # 基于zeromq作为分布式消息队列，不需要安装中间件，可以支持跨机器但不支持持久化。
 
-    RedisBrpopLpush = 'RedisBrpopLpush'  # 基于redis的list结构但是采用brpoplpush 双队列形式，和 redis_ack_able的实现差不多，实现上采用了原生命令就不需要lua脚本来实现取出和加入unack了。
+
 
     """
     操作 kombu 包，这个包也是celery的中间件依赖包，这个包可以操作10种中间件(例如rabbitmq redis)，但没包括分布式函数调度框架的kafka nsq zeromq 等。
@@ -69,7 +69,6 @@ class BrokerEnum:
 
     PEEWEE = 'PEEWEE'  # peewee包操作mysql，使用表模拟消息队列
 
-    REDIS_PUBSUB = 'REDIS_PUBSUB'  # 基于redis 发布订阅的，发布一个消息多个消费者都能收到同一条消息，但不支持持久化
 
     CELERY = 'CELERY'  # funboost支持celery框架来发布和消费任务，由celery框架来调度执行任务，但是写法简单远远暴击用户亲自使用celery的麻烦程度，
     # 用户永无无需关心和操作Celery对象实例,无需关心celery的task_routes和include配置,funboost来自动化设置这些celery配置。

funboost/consumers/base_consumer.py

@@ -30,13 +30,13 @@ from threading import Lock
 import asyncio
 
 import nb_log
-from funboost.core.current_task import funboost_current_task, get_current_taskid
+from funboost.core.current_task import funboost_current_task
 from funboost.core.loggers import develop_logger
 
 from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
 from funboost.core.task_id_logger import TaskIdLogger
-from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, CompatibleLogger,
-                    LoggerMixinDefaultWithFileHandler, stdout_write, is_main_process,
+from funboost.utils.json_helper import JsonUtils
+from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager,  is_main_process,
                     nb_log_config_default)
 from funboost.core.loggers import FunboostFileLoggerMixin, logger_prompt
 
@@ -332,7 +332,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         try:
             self._concurrent_mode_dispatcher.check_all_concurrent_mode()
             self._check_monkey_patch()
-        except BaseException:
+        except BaseException:  # noqa
             traceback.print_exc()
             os._exit(4444)  # noqa
         self.logger.info(f'开始消费 {self._queue_name} 中的消息')
@@ -384,10 +384,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         """
         raise NotImplementedError
 
-    def convert_msg_before_run(self, msg: typing.Union[str,dict]):
+    def convert_msg_before_run(self, msg: typing.Union[str, dict]) -> dict:
         """
         转换消息,消息没有使用funboost来发送,并且没有extra相关字段时候
-        用户也可以按照4.21文档,继承任意Consumer类,并实现这个方法 convert_msg_before_run,先转换消息.
+        用户也可以按照4.21文档,继承任意Consumer类,并实现这个方法 convert_msg_before_run,先转换不规范的消息.
         """
         """ 一般消息至少包含这样
         {
@@ -405,7 +405,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
                         'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
         """
-        if isinstance(msg,str):
+        if isinstance(msg, str):
             msg = json.loads(msg)
         # 以下是清洗补全字段.
         if 'extra' not in msg:
@@ -429,9 +429,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     self._last_show_pause_log_time = time.time()
             else:
                 break
-        msg = kw['body']
-        self._print_message_get_from_broker(msg)
-        kw['body'] = self.convert_msg_before_run(msg)
+        kw['body'] = self.convert_msg_before_run(kw['body'])
+        self._print_message_get_from_broker(kw['body'])
         if self._judge_is_daylight():
             self._requeue(kw)
             time.sleep(self.time_interval_for_check_do_not_run_time)
@@ -530,13 +529,11 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             if self._has_execute_times_in_recent_second >= qpsx:
                 time.sleep((1 - (time.time() - self._last_start_count_qps_timestamp)) * 1)
 
-    def _print_message_get_from_broker(self, msg,broker_name=None):
+    def _print_message_get_from_broker(self, msg, broker_name=None):
         # print(999)
         if self.consumer_params.is_show_message_get_from_broker:
-            if isinstance(msg, (dict, list)):
-                msg = json.dumps(msg, ensure_ascii=False)
             # self.logger.debug(f'从 {broker_name} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
-            self.logger.debug(f'从 {broker_name or self.consumer_params.broker_kind} 中间件 的 {self._queue_name} 中取出的消息是 {msg}')
+            self.logger.debug(f'从 {broker_name or self.consumer_params.broker_kind} 中间件 的 {self._queue_name} 中取出的消息是 {JsonUtils.to_json_str(msg)}')
 
     def _get_priority_conf(self, kw: dict, broker_task_config_key: str):
         broker_task_config = kw['body'].get('extra', {}).get(broker_task_config_key, None)
@@ -624,13 +621,15 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
                           f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
                     self.logger.info(msg)
-                    if self._msg_num_in_broker != -1 and time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
-                        # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
-                        need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
-                                                                            self._distributed_consumer_statistics.active_consumer_num)
-                        msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
-                        self.logger.info(msg)
-                        self._last_show_remaining_execution_time = time.time()
+                    if  time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:
+                        self._msg_num_in_broker = self.publisher_of_same_queue.get_message_count()
+                        if self._msg_num_in_broker != -1 :  # 有的中间件无法统计或没实现统计队列剩余数量的，统一返回的是-1，不显示这句话。
+                            # msg += f''' ，预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
+                            need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
+                                                                                self._distributed_consumer_statistics.active_consumer_num)
+                            msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.queue_name} 中的 {self._msg_num_in_broker} 个剩余任务'''
+                            self.logger.info(msg)
+                            self._last_show_remaining_execution_time = time.time()
                     self._current_time_for_execute_task_times_every_unit_time = time.time()
                     self._consuming_function_cost_time_total_every_unit_time = 0
                     self._execute_task_times_every_unit_time = 0
@@ -830,7 +829,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 self.logger.critical(msg=log_msg)
                 # noinspection PyProtectedMember,PyUnresolvedReferences
                 os._exit(444)
-            if not self.consumer_params.function_timeout :
+            if not self.consumer_params.function_timeout:
                 rs = await corotinue_obj
                 # rs = await asyncio.wait_for(corotinue_obj, timeout=4)
             else:

funboost/consumers/redis_consumer_ack_using_timeout.py

@@ -0,0 +1,77 @@
+# -*- coding: utf-8 -*-
+# @Author  : ydf
+# @Time    : 2024/8/8 0008 13:32
+import json
+import time
+from funboost.consumers.base_consumer import AbstractConsumer
+from funboost.utils.decorators import RedisDistributedLockContextManager
+from funboost.utils.json_helper import JsonUtils
+from funboost.utils.redis_manager import RedisMixin
+
+
+class RedisConsumerAckUsingTimeout(AbstractConsumer, RedisMixin):
+    """
+    redis作为中间件实现的。
+    使用超时未能ack就自动重入消息队列，例如消息取出后，由于突然断电或重启或其他原因，导致消息以后再也不能主动ack了，超过一定时间就重新放入消息队列
+    """
+
+    BROKER_EXCLUSIVE_CONFIG_DEFAULT = {'ack_timeout': 3600}
+
+    # RedisConsumerAckUsingTimeout的ack timeot 是代表消息取出后过了多少秒还未ack，就自动重回队列。这个配置一定要大于函数消耗时间，否则不停的重回队列。
+    '''用法，如何设置ack_timeout，是使用 broker_exclusive_config 中传递，就能覆盖这里的3600，用户不用改BROKER_EXCLUSIVE_CONFIG_DEFAULT的源码。
+    @boost(BoosterParams(queue_name='test_redis_ack__use_timeout', broker_kind=BrokerEnum.REIDS_ACK_USING_TIMEOUT,
+                         concurrent_num=5, log_level=20, broker_exclusive_config={'ack_timeout': 30}))
+    '''
+
+    def custom_init(self):
+        self._unack_zset_name = f'{self._queue_name}__unack_using_timeout'
+        self._ack_timeout = self.consumer_params.broker_exclusive_config['ack_timeout']
+        self._last_show_unack_ts = time.time()
+
+    def start_consuming_message(self):
+        self._is_send_consumer_hearbeat_to_redis = True
+        super().start_consuming_message()
+        self.keep_circulating(10, block=False)(self._requeue_tasks_which_unconfirmed)()
+
+    # def _add_task_str_to_unack_zset(self, task_str, ):
+    #     self.redis_db_frame.zadd(self._unack_zset_name, {task_str: time.time()})
+
+    def _confirm_consume(self, kw):
+        self.redis_db_frame.zrem(self._unack_zset_name, kw['task_str'])
+
+    def _requeue(self, kw):
+        self.redis_db_frame.rpush(self._queue_name, JsonUtils.to_json_str(kw['body']))
+
+    def _shedual_task(self):
+        lua = '''
+                     local v = redis.call("lpop", KEYS[1])
+                     if v then
+                     redis.call('zadd',KEYS[2],ARGV[1],v)
+                      end
+                     return v
+                '''
+        script = self.redis_db_frame.register_script(lua)
+        while True:
+            return_v = script(keys=[self._queue_name, self._unack_zset_name], args=[time.time()])
+            if return_v:
+                task_str = return_v
+                kw = {'body': task_str, 'task_str': task_str}
+                self._submit_task(kw)
+            else:
+                time.sleep(0.1)
+
+    def _requeue_tasks_which_unconfirmed(self):
+        """不使用这种方案，不适合本来来就需要长耗时的函数，很死板"""
+        # 防止在多个进程或多个机器中同时做扫描和放入未确认消费的任务。使用个分布式锁。
+        lock_key = f'funboost_lock__requeue_tasks_which_unconfirmed_timeout:{self._queue_name}'
+        with RedisDistributedLockContextManager(self.redis_db_frame, lock_key, ) as lock:
+            if lock.has_aquire_lock:
+                time_max = time.time() - self._ack_timeout
+                for value in self.redis_db_frame.zrangebyscore(self._unack_zset_name, 0, time_max):
+                    self.logger.warning(f'超过了 {self._ack_timeout} 秒未能确认消费, 向 {self._queue_name} 队列重新放入未消费确认的任务 {value} ,')
+                    self._requeue({'body': value})
+                    self.redis_db_frame.zrem(self._unack_zset_name, value)
+                if time.time() - self._last_show_unack_ts > 600:  # 不要频繁提示打扰
+                    self.logger.info(f'{self._unack_zset_name} 中有待确认消费任务的数量是'
+                                     f' {self.redis_db_frame.zcard(self._unack_zset_name)}')
+                    self._last_show_unack_ts = time.time()

funboost/consumers/redis_consumer_simple.py

@@ -17,7 +17,6 @@ class RedisConsumer(AbstractConsumer, RedisMixin):
         while True:
             result = self.redis_db_frame.blpop(self._queue_name,timeout=60)
             if result:
-
                 kw = {'body': result[1]}
                 self._submit_task(kw)
 

funboost/contrib/api_publish_msg.py

@@ -1,7 +1,7 @@
 import traceback
 import typing
 
-from funboost import AioAsyncResult, AsyncResult
+from funboost import AioAsyncResult, AsyncResult,PriorityConsumingControlConfig
 
 from funboost.core.cli.discovery_boosters import BoosterDiscovery
 from funboost import BoostersManager
@@ -39,11 +39,12 @@ async def publish_msg(msg_item: MsgItem):
         if msg_item.need_result:
             if booster.boost_params.is_using_rpc_mode is False:
                 raise ValueError(f' need_result 为true,{booster.queue_name} 队列消费者 需要@boost设置支持rpc模式')
-            async_result = booster.publish(msg_item.msg_body)
+            async_result = await booster.aio_publish(msg_item.msg_body,priority_control_config=PriorityConsumingControlConfig(is_using_rpc_mode=True))
             status_and_result = await AioAsyncResult(async_result.task_id, timeout=msg_item.timeout).status_and_result
+            print(status_and_result)
             # status_and_result = AsyncResult(async_result.task_id, timeout=msg_item.timeout).status_and_result
         else:
-            booster.publish(msg_item.msg_body)
+            await booster.aio_publish(msg_item.msg_body)
         return PublishResponse(succ=True, msg=f'{msg_item.queue_name} 队列,消息发布成功', status_and_result=status_and_result)
     except Exception as e:
         return PublishResponse(succ=False, msg=f'{msg_item.queue_name} 队列,消息发布失败 {type(e)} {e} {traceback.format_exc()}',

funboost/core/booster.py

@@ -4,6 +4,8 @@ import os
 import types
 import typing
 
+from funboost.concurrent_pool import FlexibleThreadPool
+from funboost.concurrent_pool.async_helper import simple_run_in_executor
 from funboost.utils.ctrl_c_end import ctrl_c_recv
 from funboost.core.loggers import flogger, develop_logger, logger_prompt
 
@@ -15,8 +17,8 @@ from funboost.core.func_params_model import BoosterParams, FunctionResultStatusP
 from funboost.factories.consumer_factory import get_consumer
 from collections import defaultdict
 
-if typing.TYPE_CHECKING:
-    from funboost.core.msg_result_getter import AsyncResult
+
+from funboost.core.msg_result_getter import AsyncResult, AioAsyncResult
 
 
 class Booster:
@@ -123,6 +125,19 @@ class Booster:
         consumer = BoostersManager.get_or_create_booster_by_queue_name(self.queue_name).consumer
         return consumer.publisher_of_same_queue.publish(msg=msg, task_id=task_id, priority_control_config=priority_control_config)
 
+    async def aio_push(self, *func_args, **func_kwargs) -> AioAsyncResult:
+        """asyncio 生态下发布消息,因为同步push只需要消耗不到1毫秒,所以基本上大概可以直接在asyncio异步生态中直接调用同步的push方法,
+        但为了更好的防止网络波动(例如发布消息到外网的消息队列耗时达到10毫秒),可以使用aio_push"""
+        async_result = await simple_run_in_executor(self.push, *func_args, **func_kwargs)
+        return AioAsyncResult(async_result.task_id, )
+
+    async def aio_publish(self, msg: typing.Union[str, dict], task_id=None,
+                      priority_control_config: PriorityConsumingControlConfig = None) -> AioAsyncResult:
+        """asyncio 生态下发布消息,因为同步push只需要消耗不到1毫秒,所以基本上大概可以直接在asyncio异步生态中直接调用同步的push方法,
+        但为了更好的防止网络波动(例如发布消息到外网的消息队列耗时达到10毫秒),可以使用aio_push"""
+        async_result = await simple_run_in_executor(self.publish,msg,task_id,priority_control_config)
+        return AioAsyncResult(async_result.task_id, )
+
     # noinspection PyMethodMayBeStatic
     def multi_process_consume(self, process_num=1):
         """超高速多进程消费"""

funboost/core/funboost_time.py

@@ -13,16 +13,33 @@ class FunboostTime(NbTime):
     def get_time_zone_str(self,time_zone: typing.Union[str, datetime.tzinfo,None] = None):
         return time_zone or self.default_time_zone or  FunboostCommonConfig.TIMEZONE  or self.get_localzone_name()
 
+    @staticmethod
+    def _get_tow_digist(num:int)->str:
+        if len(str(num)) ==1:
+            return f'0{num}'
+        return str(num)
+
+    def get_str(self, formatter=None):
+        return self.datetime_obj.strftime(formatter or self.datetime_formatter)
+
+    def get_str_fast(self):
+        t_str = f'{self.datetime_obj.year}-{self._get_tow_digist(self.datetime_obj.month)}-{self._get_tow_digist(self.datetime_obj.day)} {self._get_tow_digist(self.datetime_obj.hour)}:{self._get_tow_digist(self.datetime_obj.minute)}:{self._get_tow_digist(self.datetime_obj.second)}'
+        return t_str
 
 
 if __name__ == '__main__':
-    print(NbTime())
-    for i in range(100000):
-        # print(generate_publish_time())
-        # print(generate_publish_time_format())
-        # generate_publish_time()
-        # generate_publish_time_format()
-
-        datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).strftime(NbTime.FORMATTER_DATETIME_NO_ZONE)
-        datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).timestamp()
+    print(FunboostTime().get_str())
+    tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)
+    for i in range(1000000):
+        pass
+        # FunboostTime()#.get_str_fast()
+
+        # datetime.datetime.now().strftime(NbTime.FORMATTER_DATETIME_NO_ZONE)
+        tz = pytz.timezone(FunboostCommonConfig.TIMEZONE)
+        datetime.datetime.now(tz=tz)
+        # datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE))#.strftime(NbTime.FORMATTER_DATETIME_NO_ZONE)
+        # datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).timestamp()
+
+        # time.strftime(NbTime.FORMATTER_DATETIME_NO_ZONE)
+        # time.time()
     print(NbTime())

funboost/core/helper_funs.py

@@ -1,7 +1,8 @@
 import copy
+import pytz
 import time
 import uuid
-
+import datetime
 from funboost.core.funboost_time import FunboostTime
 
 
@@ -46,7 +47,7 @@ class MsgGenerater:
 
     @staticmethod
     def generate_publish_time() -> float:
-        return round(FunboostTime().timestamp,4)
+        return round(time.time(),4)
 
     @staticmethod
     def generate_publish_time_format() -> str:
@@ -59,3 +60,16 @@ class MsgGenerater:
         return extra_params
 
 
+
+if __name__ == '__main__':
+
+    from funboost import FunboostCommonConfig
+
+    print(FunboostTime())
+    for i in range(1000000):
+        # time.time()
+        # MsgGenerater.generate_publish_time_format()
+
+        datetime.datetime.now(tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)).strftime(FunboostTime.FORMATTER_DATETIME_NO_ZONE)
+
+    print(FunboostTime())

funboost/core/lazy_impoter.py

@@ -168,7 +168,7 @@ class KafkaPythonImporter:
 
 if __name__ == '__main__':
     print()
-    for i in range(10000000):
+    for i in range(1000000):
         # funboost_lazy_impoter.BoostersManager
         # EventletImporter().greenpool
         # GeventImporter().JoinableQueue

funboost/core/loggers.py

@@ -40,11 +40,11 @@ class FunboostMetaTypeFileLogger(type):
         cls.logger: logging.Logger = get_funboost_file_logger(name)
 
 
-flogger = get_funboost_file_logger('funboost', )
+nb_log.LogManager('_KeepAliveTimeThread').preset_log_level(_try_get_user_funboost_common_config('KEEPALIVETIMETHREAD_LOG_LEVEL') or logging.DEBUG)
 
+flogger = get_funboost_file_logger('funboost', )
 # print(_try_get_user_funboost_common_config('FUNBOOST_PROMPT_LOG_LEVEL'))
-logger_prompt = get_funboost_file_logger('funboost.prompt', log_level_int=_try_get_user_funboost_common_config('FUNBOOST_PROMPT_LOG_LEVEL') or logging.DEBUG)  # funboost框架的提示,用户自己可以设置日志级别
-nb_log.LogManager('_KeepAliveTimeThread').preset_log_level(_try_get_user_funboost_common_config('KEEPALIVETIMETHREAD_LOG_LEVEL') or logging.DEBUG)
+logger_prompt = get_funboost_file_logger('funboost.prompt', log_level_int=_try_get_user_funboost_common_config('FUNBOOST_PROMPT_LOG_LEVEL') or logging.DEBUG)
 
 # 开发时候的调试日志，比print方便通过级别一键屏蔽。
 develop_logger = get_logger('funboost_develop', log_level_int=logging.WARNING, log_filename='funboost_develop.log')

funboost/factories/broker_kind__publsiher_consumer_type_map.py

@@ -1,5 +1,6 @@
 import typing
 
+
 from funboost.publishers.http_publisher import HTTPPublisher
 from funboost.publishers.nats_publisher import NatsPublisher
 from funboost.publishers.peewee_publisher import PeeweePublisher
@@ -48,6 +49,7 @@ from funboost.consumers.udp_consumer import UDPConsumer
 from funboost.consumers.zeromq_consumer import ZeroMqConsumer
 from funboost.consumers.mqtt_consumer import MqttConsumer
 from funboost.consumers.httpsqs_consumer import HttpsqsConsumer
+from funboost.consumers.redis_consumer_ack_using_timeout import RedisConsumerAckUsingTimeout
 
 from funboost.publishers.base_publisher import AbstractPublisher
 from funboost.consumers.base_consumer import AbstractConsumer
@@ -77,6 +79,7 @@ broker_kind__publsiher_consumer_type_map = {
     BrokerEnum.TXT_FILE: (TxtFilePublisher, TxtFileConsumer),
     BrokerEnum.PEEWEE: (PeeweePublisher, PeeweeConsumer),
     BrokerEnum.REDIS_PUBSUB: (RedisPubSubPublisher, RedisPbSubConsumer),
+    BrokerEnum.REIDS_ACK_USING_TIMEOUT: (RedisPublisher, RedisConsumerAckUsingTimeout),
 
 }
 
@@ -163,8 +166,6 @@ def regist_to_funboost(broker_kind: str):
         register_custom_broker(broker_kind, NsqPublisher, NsqConsumer)
 
 
-
-
 if __name__ == '__main__':
     import sys
 

funboost/funboost_config_deafult.py

@@ -87,7 +87,7 @@ class BrokerConnConfig(DataClassBase):
     KOMBU_URL = 'redis://127.0.0.1:6379/9'  # 这个就是celery依赖包kombu使用的消息队列格式，所以funboost支持一切celery支持的消息队列种类。
     # KOMBU_URL =  'sqla+sqlite:////dssf_kombu_sqlite.sqlite'  # 4个//// 代表磁盘根目录下生成一个文件。推荐绝对路径。3个///是相对路径。
 
-    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/12'  # 使用celery作为中间件。funboost新增支持celery框架来运行函数
+    CELERY_BROKER_URL = 'redis://127.0.0.1:6379/12'  # 使用celery作为中间件。funboost新增支持celery框架来运行函数,url内容就是celery的broker形式.
     CELERY_RESULT_BACKEND = 'redis://127.0.0.1:6379/13'  # celery结果存放，可以为None
 
     DRAMATIQ_URL = RABBITMQ_URL

funboost/utils/decorators.py

@@ -332,6 +332,7 @@ class RedisDistributedLockContextManager(LoggerMixin, LoggerLevelSetterMixin):
         self._expire_seconds = expire_seconds
         self.identifier = str(uuid.uuid4())
         self.has_aquire_lock = False
+        self.logger.setLevel(logging.INFO)
 
     def __enter__(self):
         self._line = sys._getframe().f_back.f_lineno  # 调用此方法的代码的函数

funboost/utils/dependency_packages_in_pythonpath/aioredis/__init__.py

@@ -1,59 +1,59 @@
-from aioredis.client import Redis, StrictRedis
-from aioredis.connection import (
-    BlockingConnectionPool,
-    Connection,
-    ConnectionPool,
-    SSLConnection,
-    UnixDomainSocketConnection,
-)
-from aioredis.exceptions import (
-    AuthenticationError,
-    AuthenticationWrongNumberOfArgsError,
-    BusyLoadingError,
-    ChildDeadlockedError,
-    ConnectionError,
-    DataError,
-    InvalidResponse,
-    PubSubError,
-    ReadOnlyError,
-    RedisError,
-    ResponseError,
-    TimeoutError,
-    WatchError,
-)
-from aioredis.utils import from_url
-
-
-def int_or_str(value):
-    try:
-        return int(value)
-    except ValueError:
-        return value
-
-
-__version__ = "2.0.1"
-VERSION = tuple(map(int_or_str, __version__.split(".")))
-
-__all__ = [
-    "AuthenticationError",
-    "AuthenticationWrongNumberOfArgsError",
-    "BlockingConnectionPool",
-    "BusyLoadingError",
-    "ChildDeadlockedError",
-    "Connection",
-    "ConnectionError",
-    "ConnectionPool",
-    "DataError",
-    "from_url",
-    "InvalidResponse",
-    "PubSubError",
-    "ReadOnlyError",
-    "Redis",
-    "RedisError",
-    "ResponseError",
-    "SSLConnection",
-    "StrictRedis",
-    "TimeoutError",
-    "UnixDomainSocketConnection",
-    "WatchError",
-]
+from aioredis.client import Redis, StrictRedis
+from aioredis.connection import (
+    BlockingConnectionPool,
+    Connection,
+    ConnectionPool,
+    SSLConnection,
+    UnixDomainSocketConnection,
+)
+from aioredis.exceptions import (
+    AuthenticationError,
+    AuthenticationWrongNumberOfArgsError,
+    BusyLoadingError,
+    ChildDeadlockedError,
+    ConnectionError,
+    DataError,
+    InvalidResponse,
+    PubSubError,
+    ReadOnlyError,
+    RedisError,
+    ResponseError,
+    TimeoutError,
+    WatchError,
+)
+from aioredis.utils import from_url
+
+
+def int_or_str(value):
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+
+__version__ = "2.0.1"
+VERSION = tuple(map(int_or_str, __version__.split(".")))
+
+__all__ = [
+    "AuthenticationError",
+    "AuthenticationWrongNumberOfArgsError",
+    "BlockingConnectionPool",
+    "BusyLoadingError",
+    "ChildDeadlockedError",
+    "Connection",
+    "ConnectionError",
+    "ConnectionPool",
+    "DataError",
+    "from_url",
+    "InvalidResponse",
+    "PubSubError",
+    "ReadOnlyError",
+    "Redis",
+    "RedisError",
+    "ResponseError",
+    "SSLConnection",
+    "StrictRedis",
+    "TimeoutError",
+    "UnixDomainSocketConnection",
+    "WatchError",
+]