funboost 19.0__py3-none-any.whl → 19.2__py3-none-any.whl

funboost/consumers/base_consumer.py

@@ -174,8 +174,9 @@ class ResultPersistenceHelper(MongoMixin, LoggerMixin):
         self._bulk_list_lock = Lock()
         self._last_bulk_insert_time = 0
         self._has_start_bulk_insert_thread = False
+        self._queue_name = queue_name
         if self.function_result_status_persistance_conf.is_save_status:
-            task_status_col = self.mongo_db_task_status.get_collection(queue_name)
+            task_status_col = self.get_mongo_collection('task_status', queue_name)
             try:
                 # params_str 如果很长，必须使用TEXt或HASHED索引。
                 task_status_col.create_indexes([IndexModel([("insert_time_str", -1)]), IndexModel([("insert_time", -1)]),
@@ -186,11 +187,11 @@ class ResultPersistenceHelper(MongoMixin, LoggerMixin):
             except pymongo.errors.OperationFailure as e:  # 新的mongo服务端，每次启动重复创建已存在索引会报错，try一下。
                 self.logger.warning(e)
             # self._mongo_bulk_write_helper = MongoBulkWriteHelper(task_status_col, 100, 2)
-            self.task_status_col = task_status_col
             self.logger.info(f"函数运行状态结果将保存至mongo的 task_status 库的 {queue_name} 集合中，请确认 funboost.py文件中配置的 MONGO_CONNECT_URL")
 
     def save_function_result_to_mongo(self, function_result_status: FunctionResultStatus):
         if self.function_result_status_persistance_conf.is_save_status:
+            task_status_col = self.get_mongo_collection('task_status', self._queue_name)
             item = function_result_status.get_status_dict()
             item2 = copy.copy(item)
             if not self.function_result_status_persistance_conf.is_save_result:
@@ -213,12 +214,13 @@ class ResultPersistenceHelper(MongoMixin, LoggerMixin):
                                                     daemon=False)(self._bulk_insert)()
                         self.logger.warning(f'启动批量保存函数消费状态 结果到mongo的 线程')
             else:
-                self.task_status_col.insert_one(item2)  # 立即实时插入。
+                task_status_col.insert_one(item2)  # 立即实时插入。
 
     def _bulk_insert(self):
         with self._bulk_list_lock:
             if time.time() - self._last_bulk_insert_time > 0.5 and self._bulk_list:
-                self.task_status_col.bulk_write(self._bulk_list, ordered=False)
+                task_status_col = self.get_mongo_collection('task_status', self._queue_name)
+                task_status_col.bulk_write(self._bulk_list, ordered=False)
                 self._bulk_list.clear()
                 self._last_bulk_insert_time = time.time()
 
@@ -794,6 +796,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                             json.dumps(current_function_result_status.get_status_dict(without_datetime_obj=True)))
                     p.expire(kw['body']['extra']['task_id'], 600)
                     p.execute()
+
             if (current_function_result_status.success is False and current_retry_times == max_retry_times) or current_function_result_status.success is True:
                 await simple_run_in_executor(push_result)
 

funboost/function_result_web/functions.py

@@ -15,7 +15,7 @@ from funboost.utils.mongo_util import MongoMixin
 #
 # do_patch_frame_config()
 
-db = MongoMixin().mongo_db_task_status
+
 
 
 # print(db)
@@ -23,6 +23,7 @@ db = MongoMixin().mongo_db_task_status
 # print(db.list_collection_names())
 
 def get_cols(col_name_search: str):
+    db = MongoMixin().mongo_db_task_status
     if not col_name_search:
         collection_name_list = db.list_collection_names()
     else:
@@ -35,6 +36,7 @@ def get_cols(col_name_search: str):
 
 
 def query_result(col_name, start_time, end_time, is_success, function_params: str, page, ):
+    db = MongoMixin().mongo_db_task_status
     condition = {
         'insert_time': {'$gt': time_util.DatetimeConverter(start_time).datetime_obj,
                         '$lt': time_util.DatetimeConverter(end_time).datetime_obj},
@@ -59,6 +61,7 @@ def query_result(col_name, start_time, end_time, is_success, function_params: st
 
 
 def get_speed(col_name, start_time, end_time):
+    db = MongoMixin().mongo_db_task_status
     condition = {
         'insert_time': {'$gt': time_util.DatetimeConverter(start_time).datetime_obj,
                         '$lt': time_util.DatetimeConverter(end_time).datetime_obj},
@@ -78,6 +81,7 @@ def get_speed(col_name, start_time, end_time):
 
 class Statistic(LoggerMixin):
     def __init__(self, col_name):
+        db = MongoMixin().mongo_db_task_status
         self.col = db.get_collection(col_name)
         self.result = {'recent_10_days': {'time_arr': [], 'count_arr': []},
                        'recent_24_hours': {'time_arr': [], 'count_arr': []},

funboost/publishers/mongomq_publisher.py

@@ -1,6 +1,8 @@
 # -*- coding: utf-8 -*-
 # @Author  : ydf
 # @Time    : 2022/8/8 0008 12:23
+import os
+
 import json
 from funboost.utils.dependency_packages.mongomq import MongoQueue
 from funboost.publishers.base_publisher import AbstractPublisher
@@ -11,13 +13,26 @@ from funboost.utils.mongo_util import MongoMixin
 class MongoMqPublisher(AbstractPublisher, MongoMixin):
     # 使用mongo-queue包实现的基于mongodb的队列。 队列是一个col，自动存放在consume_queues库中。
     # noinspection PyAttributeOutsideInit
+
+    pid__queue_map = {}
+
     def custom_init(self):
-        self.queue = MongoQueue(
-            self.mongo_client.get_database('consume_queues').get_collection(self._queue_name),
-            consumer_id=f"consumer-{time_util.DatetimeConverter().datetime_str}",
-            timeout=600,
-            max_attempts=3,
-            ttl=24 * 3600 * 365)
+        pass
+
+    @property
+    def queue(self):
+        ''' 不能提前实例化，mongo fork进程不安全，这样是动态生成queue'''
+        pid = os.getpid()
+        if pid not in MongoMqPublisher.pid__queue_map:
+            queuex = MongoQueue(
+                # self.mongo_client.get_database('consume_queues').get_collection(self._queue_name),
+                self.get_mongo_collection('consume_queues', self._queue_name),
+                consumer_id=f"consumer-{time_util.DatetimeConverter().datetime_str}",
+                timeout=600,
+                max_attempts=3,
+                ttl=24 * 3600 * 365)
+            MongoMqPublisher.pid__queue_map[pid] =queuex
+        return MongoMqPublisher.pid__queue_map[pid]
 
     def concrete_realization_of_publish(self, msg):
         # noinspection PyTypeChecker
@@ -29,7 +44,7 @@ class MongoMqPublisher(AbstractPublisher, MongoMixin):
 
     def get_message_count(self):
         # return self.queue.size()
-        return self.queue.collection.count_documents({'status':'queued'})
+        return self.queue.collection.count_documents({'status': 'queued'})
 
     def close(self):
         pass

funboost/utils/mongo_util.py

@@ -36,9 +36,10 @@ class MongoMixin:
     """
     processid__client_map = {}
     processid__db_map = {}
+    processid__col_map = {}
 
     @property
-    def mongo_client(self):
+    def mongo_client(self) -> pymongo.MongoClient:
         pid = os.getpid()
         if pid not in MongoMixin.processid__client_map:
             MongoMixin.processid__client_map[pid] = pymongo.MongoClient(funboost_config_deafult.MONGO_CONNECT_URL, connect=False)
@@ -51,3 +52,12 @@ class MongoMixin:
             MongoMixin.processid__db_map[pid] = self.mongo_client.get_database('task_status')
         return MongoMixin.processid__db_map[pid]
 
+    def get_mongo_collection(self,database_name,colleciton_name):
+        pid = os.getpid()
+        if pid not in MongoMixin.processid__col_map:
+            MongoMixin.processid__col_map[pid] = self.mongo_client.get_database(database_name).get_collection(colleciton_name)
+        return MongoMixin.processid__col_map[pid]
+
+
+
+

{funboost-19.0.dist-info → funboost-19.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 19.0
+Version: 19.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，python函数加速器，框架包罗万象，一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数。旧名字是function_scheduling_distributed_framework
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-19.0.dist-info → funboost-19.2.dist-info}/RECORD

@@ -23,7 +23,7 @@ funboost/concurrent_pool/backup/async_pool_executor0223.py,sha256=K13RheWQiX3Fc2
 funboost/concurrent_pool/backup/async_pool_executor_back.py,sha256=-asM6fu3GLJKkkBZ1cbTSX9Iico74e6QEW6IWCpjk3M,9560
 funboost/concurrent_pool/backup/async_pool_executor_janus.py,sha256=cZ0Vm0qeT3sKcWQ6B6M6EIP1Wyk9WDHjk1TWAB1YQpc,5724
 funboost/consumers/__init__.py,sha256=ZXY_6Kut1VYNQiF5aWEgIWobsW1ht9YUP0TdRZRWFqI,126
-funboost/consumers/base_consumer.py,sha256=oxAq82FfDkLo9B7aKtFPvpG7_oInZG48kmP6eCY81Xk,84294
+funboost/consumers/base_consumer.py,sha256=CzjbZZt6HkydI5OnB2wfJJcBTM6EnrTENRBQk4HwxxA,84457
 funboost/consumers/confirm_mixin.py,sha256=C2TYSRJ2vnXAVetd7YNcQ-LQq-HZgQZSAXXzsJ6rmUY,5497
 funboost/consumers/http_consumer.py,sha256=m9-o0nNeiG7UFr6iLhpfNuMd0bQNYI38i4cw0vC8MjM,2041
 funboost/consumers/http_consumer000.py,sha256=NXOSiN1qpLAJfJkuF6SjFpWQ28YxMDULzWCBTNMwYe8,4463
@@ -62,7 +62,7 @@ funboost/factories/__init__.py,sha256=s7kKKjR1HU5eMjPD6r5b-SXTVMo1zBp2JjOAtkyt5Y
 funboost/factories/consumer_factory.py,sha256=UY063zP7GQB4kYxG1WDbn8jINy3vNSiU9SJ0ZZFaagQ,3218
 funboost/factories/publisher_factotry.py,sha256=FvpPIAH174huwb5gfrXpt_g44ZwViJi2RIfkmrHKNMY,4504
 funboost/function_result_web/app.py,sha256=xUSDBwwDA5wQVWFdFBXj9Y1s7BOh2itUw5pCZl0URMw,4841
-funboost/function_result_web/functions.py,sha256=PXwMymY9_99RY_pXC38Cd3HK63PmQTGlTnrIOs6NTI8,7203
+funboost/function_result_web/functions.py,sha256=OIPMxc4jv51qnhBxFGfTZnpMx5p4lQflPoTviDTbJUc,7345
 funboost/function_result_web/__pycache__/app.cpython-37.pyc,sha256=p-jwU7xf31KOJhmhNXqj6J79PTxjMbiTU16gAotpSEw,4045
 funboost/function_result_web/__pycache__/functions.cpython-37.pyc,sha256=KuU8DnYhFpYN0p9rdDXE9mqFuE7eKkcXHCNze3aAdOw,3921
 funboost/function_result_web/static/assets/css/custom.css,sha256=3brvjy2aBOTIXcTUK4NV6dX5wFRqx6K2aLu_jQn63jM,7674
@@ -86,7 +86,7 @@ funboost/publishers/httpsqs_publisher.py,sha256=7cf5ijwrbp4smq6ofndrKisruAqG0Wzf
 funboost/publishers/kafka_publisher.py,sha256=49JKL9PopDSIiQ3ZKBSSNPhByR-owxc2pbM4nT4Tc6g,2156
 funboost/publishers/kombu_publisher.py,sha256=ChX3qVE7Kvdu9XVFCKezV2KAwUqY7EhpvMS8nFelsY8,4567
 funboost/publishers/local_python_queue_publisher.py,sha256=veskMS5tjeneYU9HmrJLXZSK9_UT48NzHzcljjOoy3g,1365
-funboost/publishers/mongomq_publisher.py,sha256=hK2yIwfHctexmUKIjPjvbIjclc2JjzpMC15aIX1WtfI,1327
+funboost/publishers/mongomq_publisher.py,sha256=Aia8xIJc6GhFNftyYhTQq0KtLbPDR8IM3AnAm6bky4Y,1814
 funboost/publishers/mqtt_publisher.py,sha256=NKVDE5R12QL99IXgRjJtF4phyW8QaXKxHkqW5p_kXr4,3050
 funboost/publishers/msg_result_getter.py,sha256=R_tYQUgnZtTUWge--i1j_aYZLY3wysNTTFKCQ2JKQbE,6161
 funboost/publishers/nats_publisher.py,sha256=hFfaQovij9dm8w-iRN0SgiHHoS_TlrTAjw42dPwCLSA,776
@@ -123,7 +123,7 @@ funboost/utils/custom_pysnooper.py,sha256=7yXLKEMY_JjPRRt0Y0N-wV2CFhILlYNh40Y6uR
 funboost/utils/decorators.py,sha256=1JdiRucuQf_kApRO7gqxiD6K5eUuK_HjetURNRlodxQ,24272
 funboost/utils/develop_log.py,sha256=f82JHBPBjdF_By5P_Ft4wMREeIHtF8MmqGFYa_pZbyA,251
 funboost/utils/log_manager000.py,sha256=GT9rR9O-UDeaFwTCwt68xdVAmrw0vmAi1FofjRSXrJ4,73634
-funboost/utils/mongo_util.py,sha256=nh9RI-d8Nmtb2rcm3BLBkVsnf7bLTDMBG00AkEFUVis,1910
+funboost/utils/mongo_util.py,sha256=BWZnkbyHU7-zFCQ5r0Ajib5kK1S-INUR3IbE5ejIxzo,2297
 funboost/utils/monkey_color_log.py,sha256=jxxxVKebX0MmVjg75YRXUOBhPbksPBrHp2KLxCj6pT8,7357
 funboost/utils/monkey_patches.py,sha256=Q0_jKxOfFrSgrIDSuSZFrgNh6w_LRGaKAITghrIpEwI,2882
 funboost/utils/mqtt_util.py,sha256=BfCmyYwI-B8VL9499_IuYlJDCbv6ZhwyWThMf8dANOU,3199
@@ -157,8 +157,8 @@ funboost/utils/pysnooper_ydf/pycompat.py,sha256=ehsCfjsLdwoK0_o5fwYWDo3WeqCVfHW5
 funboost/utils/pysnooper_ydf/tracer.py,sha256=2leNcEhT0qt2B_7Vn094dep9AG0qnRX94imQHm4Qp8k,19122
 funboost/utils/pysnooper_ydf/utils.py,sha256=Y0nGXPqdiHBwO93TQAyYNbrzXtcK3_BQXrDVyjcTqVw,2748
 funboost/utils/pysnooper_ydf/variables.py,sha256=N12-WKrg4GFt3X1aw9c-jwuVbn2wQTNZXscrYP9RM2k,3678
-funboost-19.0.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-funboost-19.0.dist-info/METADATA,sha256=aQffevvOr0Og3ksMq0J3AQfd7kDjJpX4HW0mEzvFuns,25630
-funboost-19.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
-funboost-19.0.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
-funboost-19.0.dist-info/RECORD,,
+funboost-19.2.dist-info/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+funboost-19.2.dist-info/METADATA,sha256=_8V8rpfUTACU5cBPRhdxHkF_Nduoi8X3swyry2mQ1hA,25630
+funboost-19.2.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92
+funboost-19.2.dist-info/top_level.txt,sha256=K8WuKnS6MRcEWxP1NvbmCeujJq6TEfbsB150YROlRw0,9
+funboost-19.2.dist-info/RECORD,,