funboost 18.9__py3-none-any.whl → 19.1__py3-none-any.whl

funboost/consumers/base_consumer.py

@@ -106,6 +106,8 @@ class FunctionResultStatus(LoggerMixin, LoggerLevelSetterMixin):
     script_name_long = sys.argv[0]
     script_name = script_name_long.split('/')[-1].split('\\')[-1]
 
+    FUNC_RUN_ERROR = 'FUNC_RUN_ERROR'
+
     def __init__(self, queue_name: str, fucntion_name: str, msg_dict: dict):
         # print(params)
         self.queue_name = queue_name
@@ -172,8 +174,9 @@ class ResultPersistenceHelper(MongoMixin, LoggerMixin):
         self._bulk_list_lock = Lock()
         self._last_bulk_insert_time = 0
         self._has_start_bulk_insert_thread = False
+        self._queue_name = queue_name
         if self.function_result_status_persistance_conf.is_save_status:
-            task_status_col = self.mongo_db_task_status.get_collection(queue_name)
+            task_status_col = self.get_mongo_collection('task_status', queue_name)
             try:
                 # params_str 如果很长，必须使用TEXt或HASHED索引。
                 task_status_col.create_indexes([IndexModel([("insert_time_str", -1)]), IndexModel([("insert_time", -1)]),
@@ -184,11 +187,11 @@ class ResultPersistenceHelper(MongoMixin, LoggerMixin):
             except pymongo.errors.OperationFailure as e:  # 新的mongo服务端，每次启动重复创建已存在索引会报错，try一下。
                 self.logger.warning(e)
             # self._mongo_bulk_write_helper = MongoBulkWriteHelper(task_status_col, 100, 2)
-            self.task_status_col = task_status_col
             self.logger.info(f"函数运行状态结果将保存至mongo的 task_status 库的 {queue_name} 集合中，请确认 funboost.py文件中配置的 MONGO_CONNECT_URL")
 
     def save_function_result_to_mongo(self, function_result_status: FunctionResultStatus):
         if self.function_result_status_persistance_conf.is_save_status:
+            task_status_col = self.get_mongo_collection('task_status', self._queue_name)
             item = function_result_status.get_status_dict()
             item2 = copy.copy(item)
             if not self.function_result_status_persistance_conf.is_save_result:
@@ -211,12 +214,13 @@ class ResultPersistenceHelper(MongoMixin, LoggerMixin):
                                                     daemon=False)(self._bulk_insert)()
                         self.logger.warning(f'启动批量保存函数消费状态 结果到mongo的 线程')
             else:
-                self.task_status_col.insert_one(item2)  # 立即实时插入。
+                task_status_col.insert_one(item2)  # 立即实时插入。
 
     def _bulk_insert(self):
         with self._bulk_list_lock:
             if time.time() - self._last_bulk_insert_time > 0.5 and self._bulk_list:
-                self.task_status_col.bulk_write(self._bulk_list, ordered=False)
+                task_status_col = self.get_mongo_collection('task_status', self._queue_name)
+                task_status_col.bulk_write(self._bulk_list, ordered=False)
                 self._bulk_list.clear()
                 self._last_bulk_insert_time = time.time()
 
@@ -689,13 +693,14 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 f'函数 {self.consuming_function.__name__} 达到最大重试次数 {self._get_priority_conf(kw, "max_retry_times")} 后,仍然失败， 入参是  {function_only_params} ')
         if self._get_priority_conf(kw, 'is_using_rpc_mode'):
             # print(function_result_status.get_status_dict(without_datetime_obj=
-            with RedisMixin().redis_db_filter_and_rpc_result.pipeline() as p:
-                # RedisMixin().redis_db_frame.lpush(kw['body']['extra']['task_id'], json.dumps(function_result_status.get_status_dict(without_datetime_obj=True)))
-                # RedisMixin().redis_db_frame.expire(kw['body']['extra']['task_id'], 600)
-                p.lpush(kw['body']['extra']['task_id'],
-                        json.dumps(current_function_result_status.get_status_dict(without_datetime_obj=True)))
-                p.expire(kw['body']['extra']['task_id'], 600)
-                p.execute()
+            if (current_function_result_status.success is False and current_retry_times == max_retry_times) or current_function_result_status.success is True:
+                with RedisMixin().redis_db_filter_and_rpc_result.pipeline() as p:
+                    # RedisMixin().redis_db_frame.lpush(kw['body']['extra']['task_id'], json.dumps(function_result_status.get_status_dict(without_datetime_obj=True)))
+                    # RedisMixin().redis_db_frame.expire(kw['body']['extra']['task_id'], 600)
+                    p.lpush(kw['body']['extra']['task_id'],
+                            json.dumps(current_function_result_status.get_status_dict(without_datetime_obj=True)))
+                    p.expire(kw['body']['extra']['task_id'], 600)
+                    p.execute()
 
         with self._lock_for_count_execute_task_times_every_unit_time:
             self._execute_task_times_every_unit_time += 1
@@ -753,6 +758,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                               exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
             # traceback.print_exc()
             function_result_status.exception = f'{e.__class__.__name__}    {str(e)}'
+            function_result_status.result = FunctionResultStatus.FUNC_RUN_ERROR
         return function_result_status
 
     async def _async_run(self, kw: dict, ):
@@ -791,7 +797,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                     p.expire(kw['body']['extra']['task_id'], 600)
                     p.execute()
 
-            await simple_run_in_executor(push_result)
+            if (current_function_result_status.success is False and current_retry_times == max_retry_times) or current_function_result_status.success is True:
+                await simple_run_in_executor(push_result)
 
         # 异步执行不存在线程并发，不需要加锁。
         self._execute_task_times_every_unit_time += 1
@@ -856,6 +863,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                               f'函数运行时间是 {round(time.time() - t_start, 4)} 秒,\n  入参是  {function_only_params}    \n 原因是 {type(e)} {e} ',
                               exc_info=self._get_priority_conf(kw, 'is_print_detail_exception'))
             function_result_status.exception = f'{e.__class__.__name__}    {str(e)}'
+            function_result_status.result = FunctionResultStatus.FUNC_RUN_ERROR
         return function_result_status
 
     @abc.abstractmethod

funboost/function_result_web/functions.py

@@ -15,7 +15,7 @@ from funboost.utils.mongo_util import MongoMixin
 #
 # do_patch_frame_config()
 
-db = MongoMixin().mongo_db_task_status
+
 
 
 # print(db)
@@ -23,6 +23,7 @@ db = MongoMixin().mongo_db_task_status
 # print(db.list_collection_names())
 
 def get_cols(col_name_search: str):
+    db = MongoMixin().mongo_db_task_status
     if not col_name_search:
         collection_name_list = db.list_collection_names()
     else:
@@ -35,6 +36,7 @@ def get_cols(col_name_search: str):
 
 
 def query_result(col_name, start_time, end_time, is_success, function_params: str, page, ):
+    db = MongoMixin().mongo_db_task_status
     condition = {
         'insert_time': {'$gt': time_util.DatetimeConverter(start_time).datetime_obj,
                         '$lt': time_util.DatetimeConverter(end_time).datetime_obj},
@@ -59,6 +61,7 @@ def query_result(col_name, start_time, end_time, is_success, function_params: st
 
 
 def get_speed(col_name, start_time, end_time):
+    db = MongoMixin().mongo_db_task_status
     condition = {
         'insert_time': {'$gt': time_util.DatetimeConverter(start_time).datetime_obj,
                         '$lt': time_util.DatetimeConverter(end_time).datetime_obj},
@@ -78,6 +81,7 @@ def get_speed(col_name, start_time, end_time):
 
 class Statistic(LoggerMixin):
     def __init__(self, col_name):
+        db = MongoMixin().mongo_db_task_status
         self.col = db.get_collection(col_name)
         self.result = {'recent_10_days': {'time_arr': [], 'count_arr': []},
                        'recent_24_hours': {'time_arr': [], 'count_arr': []},

funboost/publishers/mongomq_publisher.py

@@ -12,8 +12,13 @@ class MongoMqPublisher(AbstractPublisher, MongoMixin):
     # 使用mongo-queue包实现的基于mongodb的队列。 队列是一个col，自动存放在consume_queues库中。
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        self.queue = MongoQueue(
-            self.mongo_client.get_database('consume_queues').get_collection(self._queue_name),
+        pass
+
+    @property
+    def queue(self):
+        return MongoQueue(
+            # self.mongo_client.get_database('consume_queues').get_collection(self._queue_name),
+            self.get_mongo_collection('consume_queues', self._queue_name),
             consumer_id=f"consumer-{time_util.DatetimeConverter().datetime_str}",
             timeout=600,
             max_attempts=3,
@@ -29,7 +34,7 @@ class MongoMqPublisher(AbstractPublisher, MongoMixin):
 
     def get_message_count(self):
         # return self.queue.size()
-        return self.queue.collection.count_documents({'status':'queued'})
+        return self.queue.collection.count_documents({'status': 'queued'})
 
     def close(self):
         pass

funboost/utils/dependency_packages/aioredis_adapt_py311/__init__.py

@@ -0,0 +1,59 @@
+from funboost.utils.dependency_packages.aioredis_adapt_py311.client import Redis, StrictRedis
+from funboost.utils.dependency_packages.aioredis_adapt_py311.connection import (
+    BlockingConnectionPool,
+    Connection,
+    ConnectionPool,
+    SSLConnection,
+    UnixDomainSocketConnection,
+)
+from funboost.utils.dependency_packages.aioredis_adapt_py311.exceptions import (
+    AuthenticationError,
+    AuthenticationWrongNumberOfArgsError,
+    BusyLoadingError,
+    ChildDeadlockedError,
+    ConnectionError,
+    DataError,
+    InvalidResponse,
+    PubSubError,
+    ReadOnlyError,
+    RedisError,
+    ResponseError,
+    TimeoutError,
+    WatchError,
+)
+from funboost.utils.dependency_packages.aioredis_adapt_py311.utils import from_url
+
+
+def int_or_str(value):
+    try:
+        return int(value)
+    except ValueError:
+        return value
+
+
+__version__ = "2.0.1"
+VERSION = tuple(map(int_or_str, __version__.split(".")))
+
+__all__ = [
+    "AuthenticationError",
+    "AuthenticationWrongNumberOfArgsError",
+    "BlockingConnectionPool",
+    "BusyLoadingError",
+    "ChildDeadlockedError",
+    "Connection",
+    "ConnectionError",
+    "ConnectionPool",
+    "DataError",
+    "from_url",
+    "InvalidResponse",
+    "PubSubError",
+    "ReadOnlyError",
+    "Redis",
+    "RedisError",
+    "ResponseError",
+    "SSLConnection",
+    "StrictRedis",
+    "TimeoutError",
+    "UnixDomainSocketConnection",
+    "WatchError",
+]