funboost 50.0__py3-none-any.whl → 50.2__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "50.0"
+__version__ = "50.2"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/consumers/base_consumer.py

@@ -168,8 +168,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         # if  self.consumer_params.concurrent_mode == ConcurrentModeEnum.ASYNC and self.consumer_params.specify_async_loop is None:
         #     self.consumer_params.specify_async_loop= get_or_create_event_loop()
         self._lock_for_count_execute_task_times_every_unit_time = Lock()
-        if self.consumer_params.concurrent_mode == ConcurrentModeEnum.ASYNC:
-            self._async_lock_for_count_execute_task_times_every_unit_time = asyncio.Lock()
+        
         # self._unit_time_for_count = 10  # 每隔多少秒计数，显示单位时间内执行多少次，暂时固定为10秒。
         # self._execute_task_times_every_unit_time = 0  # 每单位时间执行了多少次任务。
         # self._execute_task_times_every_unit_time_fail =0  # 每单位时间执行了多少次任务失败。
@@ -491,7 +490,6 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         return msg
 
     def _submit_task(self, kw):
-
         kw['body'] = self._convert_msg_before_run(kw['body'])
         self._print_message_get_from_broker(kw['body'])
         if self._judge_is_daylight():
@@ -499,6 +497,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             time.sleep(self.time_interval_for_check_do_not_run_time)
             return
         function_only_params = delete_keys_and_return_new_dict(kw['body'], )
+        kw['function_only_params'] = function_only_params
         if self._get_priority_conf(kw, 'do_task_filtering') and self._redis_filter.check_value_exists(
                 function_only_params,self._get_priority_conf(kw, 'filter_str')):  # 对函数的参数进行检查，过滤已经执行过并且成功的任务。
             self.logger.warning(f'redis的 [{self._redis_filter_key_name}] 键 中 过滤任务 {kw["body"]}')
@@ -688,7 +687,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             max_retry_times = self._get_priority_conf(kw, 'max_retry_times')
             current_function_result_status = FunctionResultStatus(self.queue_name, self.consuming_function.__name__, kw['body'], )
             current_retry_times = 0
-            function_only_params = delete_keys_and_return_new_dict(kw['body'])
+            function_only_params = kw['function_only_params']
             for current_retry_times in range(max_retry_times + 1):
                 current_function_result_status.run_times = current_retry_times + 1
                 current_function_result_status.run_status = RunStatus.running
@@ -747,10 +746,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
     # noinspection PyProtectedMember
     def _run_consuming_function_with_confirm_and_retry(self, kw: dict, current_retry_times,
                                                        function_result_status: FunctionResultStatus, ):
-        function_only_params = delete_keys_and_return_new_dict(kw['body']) if self._do_not_delete_extra_from_msg is False else kw['body']
+        function_only_params = kw['function_only_params'] if self._do_not_delete_extra_from_msg is False else kw['body']
         task_id = kw['body']['extra']['task_id']
         t_start = time.time()
-        # function_result_status.run_times = current_retry_times + 1
+      
         fct = funboost_current_task()
         fct_context = FctContext(function_params=function_only_params,
                                  full_msg=kw['body'],
@@ -837,17 +836,47 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
             function_result_status.result = FunctionResultStatus.FUNC_RUN_ERROR
         return function_result_status
-
+    
+    def _gen_asyncio_objects(self):
+        if getattr(self, '_async_lock_for_count_execute_task_times_every_unit_time', None) is None:
+            self._async_lock_for_count_execute_task_times_every_unit_time = asyncio.Lock()
+        
     # noinspection PyProtectedMember
     async def _async_run(self, kw: dict, ):
-        # """虽然和上面有点大面积重复相似，这个是为了asyncio模式的，asyncio模式真的和普通同步模式的代码思维和形式区别太大，
-        # 框架实现兼容async的消费函数很麻烦复杂，连并发池都要单独写"""
+        """
+        虽然 async def _async_run 和上面的 def _run 有点大面积结构重复相似，这个是为了asyncio模式的，
+        asyncio模式真的和普通同步模式的代码思维和形式区别太大，
+        框架实现兼容async的消费函数很麻烦复杂，连并发池都要单独写
+
+        _run 和 _async_run 无法合并成一个方法：
+        因为在一个函数体内部，您无法根据条件来决定是否使用 await。
+
+        Python 语法不允许这样做：
+        # 伪代码，这是无效的
+        def _unified_run(self, kw, is_async):
+            # ...
+            if is_async:
+                await asyncio.sleep(1) # 'await' outside async function 经典报错
+            else:
+                time.sleep(1)
+
+        不能在同步函数里面去写 await,只要一个函数里出现了 await，这个函数就必须被声明为 async def
+
+
+
+        funboost 这个代价算小了,为了支持异步的全流程生态包括发布/消费/获取rpc结果,对asyncio的累计专门投入代码不到500行.
+        如果是celery 改造适配asyncio,起码要增加10倍以上的代码量,改5000行代码都搞不定支持真asyncio并发.
+        我说的是支持兼容真asyncio并发,而不是每个线程内部搞个临时loop,然后临时loop.run_until_complete(用户async函数) 这种伪asyncio并发,
+        真asyncio并发,是单个loop里面运行无数协程,
+        伪asyncio并发是在每个线程启动一个临时的loop,每个loop仅仅运行一个协程,然后等待这个协程结束,这完全违背了 asyncio 的核心初心理念,这种比多线程性能本身还差.
+        """
         try:
+            self._gen_asyncio_objects()
             t_start_run_fun = time.time()
             max_retry_times = self._get_priority_conf(kw, 'max_retry_times')
             current_function_result_status = FunctionResultStatus(self.queue_name, self.consuming_function.__name__, kw['body'], )
             current_retry_times = 0
-            function_only_params = delete_keys_and_return_new_dict(kw['body'])
+            function_only_params = kw['function_only_params']
             for current_retry_times in range(max_retry_times + 1):
                 current_function_result_status.run_times = current_retry_times + 1
                 current_function_result_status.run_status = RunStatus.running
@@ -888,11 +917,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
 
                 if (current_function_result_status.success is False and current_retry_times == max_retry_times) or current_function_result_status.success is True:
                     await simple_run_in_executor(push_result)
-
             async with self._async_lock_for_count_execute_task_times_every_unit_time:
                 self.metric_calculation.cal(t_start_run_fun, current_function_result_status)
 
-            self._frame_custom_record_process_info_func(current_function_result_status)
+            self._frame_custom_record_process_info_func(current_function_result_status,kw)
             self.user_custom_record_process_info_func(current_function_result_status,)  # 两种方式都可以自定义,记录结果.建议使用文档4.21.b的方式继承来重写
             await self.aio_user_custom_record_process_info_func(current_function_result_status,)
             if self.consumer_params.user_custom_record_process_info_func:
@@ -911,8 +939,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                                                                    function_result_status: FunctionResultStatus, ):
         """虽然和上面有点大面积重复相似，这个是为了asyncio模式的，asyncio模式真的和普通同步模式的代码思维和形式区别太大，
         框架实现兼容async的消费函数很麻烦复杂，连并发池都要单独写"""
-        function_only_params = delete_keys_and_return_new_dict(kw['body']) if self._do_not_delete_extra_from_msg is False else kw['body']
-        function_result_status.run_times = current_retry_times + 1
+        function_only_params = kw['function_only_params'] if self._do_not_delete_extra_from_msg is False else kw['body']
+      
         # noinspection PyBroadException
         t_start = time.time()
         fct = funboost_current_task()

funboost/consumers/faststream_consumer.py

@@ -9,6 +9,7 @@ from faststream import FastStream,Context
 from faststream.annotations import Logger
 
 from funboost.concurrent_pool.async_helper import simple_run_in_executor
+from funboost.core.serialization import Serialization
 from funboost.core.helper_funs import delete_keys_and_return_new_dict
 
 

funboost/core/active_cousumer_info_getter.py

@@ -205,12 +205,13 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
         def _inner():
             while True:
                 t_start = time.time()
-                # 这个函数确保只有一个地方在上报数据，避免重复上报
+                # 这个函数确保只有一个地方在上报数据，避免重复采集上报
                 report_ts = self.timestamp()
                 redis_report_uuid_ts_str = self.redis_db_frame.get(RedisKeys.FUNBOOST_LAST_GET_QUEUE_PARAMS_AND_ACTIVE_CONSUMERS_AND_REPORT__UUID_TS, )
                 if redis_report_uuid_ts_str:
                     redis_report_uuid_ts = Serialization.to_dict(redis_report_uuid_ts_str)
                     if redis_report_uuid_ts['report_uuid'] != report_uuid and redis_report_uuid_ts['report_ts'] > report_ts - time_interval - 10 :
+                        time.sleep(5) # 防止cpu空转
                         continue
                 self.redis_db_frame.set(RedisKeys.FUNBOOST_LAST_GET_QUEUE_PARAMS_AND_ACTIVE_CONSUMERS_AND_REPORT__UUID_TS,
                                         Serialization.to_json_str({'report_uuid':report_uuid, 'report_ts':report_ts}))
@@ -229,17 +230,38 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
                         RedisKeys.gen_funboost_queue_time_series_data_key_by_queue_name(queue),
                         0, report_ts - 86400
                     )
-                self.logger.info(f'上报时序数据耗时 {time.time() - t_start} 秒')
+                self.logger.info(f'采集上报时序数据耗时 {time.time() - t_start} 秒')
 
                 time.sleep(time_interval)
         threading.Thread(target=_inner, daemon=daemon).start()
 
-    def get_time_series_data_by_queue_name(self,queue_name,start_ts=None,end_ts=None):
+    def get_time_series_data_by_queue_name(self,queue_name,start_ts=None,end_ts=None,curve_samples_count=None):
         res = self.redis_db_frame.zrangebyscore(
             RedisKeys.gen_funboost_queue_time_series_data_key_by_queue_name(queue_name),
             max(float(start_ts or 0),self.timestamp() - 86400) ,float(end_ts or -1),withscores=True)
         # print(res)
-        return [{'report_data':Serialization.to_dict(item[0]),'report_ts':item[1]} for item in res]
+        series_data_all= [{'report_data':Serialization.to_dict(item[0]),'report_ts':item[1]} for item in res]
+        if curve_samples_count is None:
+            return series_data_all
+        
+        # 曲线采样数量
+        total_count = len(series_data_all)
+        if total_count <= curve_samples_count:
+            # 如果原始数据量小于等于需要的样本数，直接返回全部数据
+            return series_data_all
+        
+        # 计算采样步长
+        step = total_count / curve_samples_count
+        sampled_data = []
+        
+        # 按照步长进行采样
+        for i in range(curve_samples_count):
+            index = int(i * step)
+            if index < total_count:
+                sampled_data.append(series_data_all[index])
+        
+        return sampled_data
+        
 
 if __name__ == '__main__':
     # print(Serialization.to_json_str(QueueConusmerParamsGetter().get_queue_params_and_active_consumers()))

funboost/core/funboost_time.py

@@ -1,6 +1,6 @@
 import pytz
 import time
-
+import sys
 import datetime
 
 import typing
@@ -28,6 +28,46 @@ class FunboostTime(NbTime):
         return t_str
 
 
+
+
+# 缓存时区对象，提升性能（避免重复解析）
+_tz_cache = {}
+
+def get_now_time_str_by_tz(tz_name: str=None) -> str:                   
+    """
+    根据时区名（如 'Asia/Shanghai'）返回当前时间字符串，格式：'%Y-%m-%d %H:%M:%S'
+    
+    兼容 Python 3.6+，优先使用 zoneinfo（3.9+），否则尝试 pytz
+    
+    :param tz_name: IANA 时区名称，如 'Asia/Shanghai', 'America/New_York'
+    :return: 格式化时间字符串
+    """
+    # 检查缓存
+    tz_name = tz_name or FunboostCommonConfig.TIMEZONE
+    if tz_name not in _tz_cache:
+        if sys.version_info >= (3, 9):
+            from zoneinfo import ZoneInfo
+            _tz_cache[tz_name] = ZoneInfo(tz_name)
+        else:
+            # Python < 3.9，使用 pytz
+            try:
+                import pytz
+                _tz_cache[tz_name] = pytz.timezone(tz_name)
+            except ImportError:
+                raise RuntimeError(
+                    f"Python < 3.9 requires 'pytz' to handle timezones. "
+                    f"Install it with: pip install pytz"
+                ) from None
+            except pytz.UnknownTimeZoneError:
+                raise pytz.UnknownTimeZoneError(tz_name)
+
+    tz = _tz_cache[tz_name]
+    
+    # 获取当前时间并格式化（注意：datetime.now(tz) 是最高效的方式）
+    now = datetime.datetime.now(tz)
+    return f'{now.year:04d}-{now.month:02d}-{now.day:02d} {now.hour:02d}:{now.minute:02d}:{now.second:02d}'
+    # return now.strftime("%Y-%m-%d %H:%M:%S")
+
 if __name__ == '__main__':
     print(FunboostTime().get_str())
     tz=pytz.timezone(FunboostCommonConfig.TIMEZONE)

funboost/core/function_result_status_saver.py

@@ -14,7 +14,7 @@ import sys
 from pymongo import IndexModel, ReplaceOne
 
 from funboost.core.func_params_model import FunctionResultStatusPersistanceConfig
-from funboost.core.helper_funs import get_publish_time, delete_keys_and_return_new_dict
+from funboost.core.helper_funs import get_publish_time, delete_keys_and_return_new_dict, get_publish_time_format
 from funboost.core.serialization import Serialization
 from funboost.utils import time_util, decorators
 from funboost.utils.mongo_util import MongoMixin
@@ -42,9 +42,9 @@ class FunctionResultStatus():
         self.task_id = self.msg_dict.get('extra', {}).get('task_id', '')
         self.process_id = os.getpid()
         self.thread_id = threading.get_ident()
-        self.publish_time = publish_time = get_publish_time(msg_dict)
-        if publish_time:
-            self.publish_time_str = time_util.DatetimeConverter(publish_time).datetime_str
+        self.publish_time  = get_publish_time(msg_dict)
+        self.publish_time_format = get_publish_time_format(msg_dict)
+        # print(self.publish_time_format)
         function_params = delete_keys_and_return_new_dict(msg_dict, )
         self.params = function_params
         self.params_str = Serialization.to_json_str(function_params)

funboost/core/helper_funs.py

@@ -3,7 +3,7 @@ import pytz
 import time
 import uuid
 import datetime
-from funboost.core.funboost_time import FunboostTime
+from funboost.core.funboost_time import FunboostTime, get_now_time_str_by_tz
 
 
 def get_publish_time(paramsx: dict):
@@ -14,6 +14,14 @@ def get_publish_time(paramsx: dict):
     return paramsx.get('extra', {}).get('publish_time', None)
 
 
+def get_publish_time_format(paramsx: dict):
+    """
+    :param paramsx:
+    :return:
+    """
+    return paramsx.get('extra', {}).get('publish_time_format', None)
+
+
 def delete_keys_and_return_new_dict(dictx: dict, keys: list = None):
     dict_new = copy.deepcopy(dictx)  # 主要是去掉一级键 publish_time，浅拷贝即可。新的消息已经不是这样了。
     keys = ['publish_time', 'publish_time_format', 'extra'] if keys is None else keys
@@ -49,13 +57,18 @@ class MsgGenerater:
     def generate_publish_time() -> float:
         return round(time.time(),4)
 
+    # @staticmethod  # 性能不好
+    # def generate_publish_time_format() -> str:
+    #     return FunboostTime().get_str()
+    
     @staticmethod
     def generate_publish_time_format() -> str:
-        return FunboostTime().get_str()
+        return get_now_time_str_by_tz()
 
     @classmethod
     def generate_pulish_time_and_task_id(cls,queue_name:str,task_id=None):
-        extra_params = {'task_id': task_id or cls.generate_task_id(queue_name), 'publish_time': cls.generate_publish_time(),
+        extra_params = {'task_id': task_id or cls.generate_task_id(queue_name), 
+                        'publish_time': cls.generate_publish_time(),
                         'publish_time_format': cls.generate_publish_time_format()}
         return extra_params
 

funboost/function_result_web/app.py

@@ -271,8 +271,27 @@ def get_time_series_data_by_queue_name(queue_name,):
     
     返回例如  [{'report_data': {'pause_flag': -1, 'msg_num_in_broker': 936748, 'history_run_count': '150180', 'history_run_fail_count': '46511', 'all_consumers_last_x_s_execute_count': 7, 'all_consumers_last_x_s_execute_count_fail': 0, 'all_consumers_last_x_s_avarage_function_spend_time': 3.441, 'all_consumers_avarage_function_spend_time_from_start': 4.598, 'all_consumers_total_consume_count_from_start': 1296, 'all_consumers_total_consume_count_from_start_fail': 314, 'report_ts': 1749617360.597841}, 'report_ts': 1749617360.597841}, {'report_data': {'pause_flag': -1, 'msg_num_in_broker': 936748, 'history_run_count': '150184', 'history_run_fail_count': '46514', 'all_consumers_last_x_s_execute_count': 7, 'all_consumers_last_x_s_execute_count_fail': 0, 'all_consumers_last_x_s_avarage_function_spend_time': 3.441, 'all_consumers_avarage_function_spend_time_from_start': 4.599, 'all_consumers_total_consume_count_from_start': 1299, 'all_consumers_total_consume_count_from_start_fail': 316, 'report_ts': 1749617370.628166}, 'report_ts': 1749617370.628166}] 
     """
+    # 获取前端传递的参数
+    start_ts = request.args.get('start_ts')
+    end_ts = request.args.get('end_ts')
+    curve_samples_count = request.args.get('curve_samples_count')
+    
+    # 如果前端指定了采样点数，使用前端的值
+    if curve_samples_count:
+        try:
+            curve_samples_count = int(curve_samples_count)
+            # 验证值是否在允许的范围内
+            allowed_values = [60, 120, 180, 360, 720, 1440,8640]
+            if curve_samples_count not in allowed_values:
+                curve_samples_count = 360  # 默认值
+        except (ValueError, TypeError):
+            curve_samples_count = 360  # 默认值
+    else:
+        # 如果前端没有指定，使用默认值
+        curve_samples_count = 360
+        
     return jsonify(QueueConusmerParamsGetter().get_time_series_data_by_queue_name(
-        queue_name,request.args.get('start_ts'),request.args.get('end_ts')))
+        queue_name, start_ts, end_ts, curve_samples_count))
         
 @app.route('/rpc/rpc_call',methods=['POST'])
 def rpc_call():