funboost 48.7__py3-none-any.whl → 48.8__py3-none-any.whl

funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "48.7"
+__version__ = "48.8"
 
 from funboost.set_frame_config import show_frame_config
 

funboost/__main__.py

@@ -18,6 +18,7 @@ def _check_pass_params():
 
 def main():
     _check_pass_params()
+    
 
     fire.Fire(BoosterFire, )
 
@@ -31,4 +32,5 @@ python -m funboost  --project_root_path=/codes/funboost   --booster_dirs_str=tes
 python -m funboost  --project_root_path=/codes/funboost   --booster_dirs_str=test_frame/test_funboost_cli/test_find_boosters --max_depth=2  push test_find_queue1 --x=1 --y=2
 
 
+python -m funboost  --project_root_path=/codes/funboost  start_web
 '''

funboost/constant.py

@@ -112,4 +112,5 @@ class ConstStrForClassMethod:
 class RedisKeys:
     REDIS_KEY_PAUSE_FLAG  = 'funboost_pause_flag' 
     REDIS_KEY_STOP_FLAG = 'funboost_stop_flag'
-    QUEUE__MSG_COUNT_MAP = 'funboost_queue__msg_count_map'
+    QUEUE__MSG_COUNT_MAP = 'funboost_queue__msg_count_map'
+    FUNBOOST_QUEUE__CONSUMER_PARAMS= 'funboost_queue__consumer_parmas'

funboost/consumers/base_consumer.py

@@ -165,6 +165,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         self._redis_filter = filter_class(self._redis_filter_key_name, consumer_params.task_filtering_expire_seconds)
 
         self._lock_for_count_execute_task_times_every_unit_time = Lock()
+        self._async_lock_for_count_execute_task_times_every_unit_time = asyncio.Lock()
         # self._unit_time_for_count = 10  # 每隔多少秒计数，显示单位时间内执行多少次，暂时固定为10秒。
         # self._execute_task_times_every_unit_time = 0  # 每单位时间执行了多少次任务。
         # self._execute_task_times_every_unit_time_fail =0  # 每单位时间执行了多少次任务失败。
@@ -836,8 +837,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 if (current_function_result_status.success is False and current_retry_times == max_retry_times) or current_function_result_status.success is True:
                     await simple_run_in_executor(push_result)
 
-            # 异步执行不存在线程并发，不需要加锁。
-            self.metric_calculation.cal(t_start_run_fun, current_function_result_status)
+            async with self._async_lock_for_count_execute_task_times_every_unit_time:
+                self.metric_calculation.cal(t_start_run_fun, current_function_result_status)
 
             self.user_custom_record_process_info_func(current_function_result_status)  # 两种方式都可以自定义,记录结果.建议使用文档4.21.b的方式继承来重写
             await self.aio_user_custom_record_process_info_func(current_function_result_status)
@@ -1175,7 +1176,7 @@ class MetricCalculation:
                                                                         self.consumer._distributed_consumer_statistics.active_consumer_num)
                     msg += f''' 预计还需要 {need_time} 时间 才能执行完成 队列 {self.consumer.queue_name} 中的 {self.msg_num_in_broker} 个剩余任务'''
                     self.consumer.logger.info(msg)
-                    self.last_show_remaining_execution_time = time.time()
+                self.last_show_remaining_execution_time = time.time()
             self.current_time_for_execute_task_times_every_unit_time = time.time()
             self.consuming_function_cost_time_total_every_unit_time_tmp = 0
             self.execute_task_times_every_unit_time_temp = 0

funboost/consumers/rabbitmq_amqpstorm_consumer.py

@@ -42,5 +42,5 @@ class RabbitmqConsumerAmqpStorm(AbstractConsumer):
 
     def _requeue(self, kw):
         # amqpstorm.Message.delivery_tag
-        print(kw['amqpstorm_message'].delivery_tag)
+        # print(kw['amqpstorm_message'].delivery_tag)
         kw['amqpstorm_message'].nack(requeue=True)

funboost/core/active_cousumer_info_getter.py

@@ -104,12 +104,12 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
         queue__pause_map = self.redis_db_frame.hgetall(RedisKeys.REDIS_KEY_PAUSE_FLAG)
         return {k:int(v)  for k,v in queue__pause_map.items()}
 
-    def get_msg_num(self):
+    def get_msg_num(self,ignore_report_ts=False):
         queue__msg_count_info_map = self.redis_db_frame.hgetall(RedisKeys.QUEUE__MSG_COUNT_MAP)
         queue__msg_count_dict = {}
         for queue_name,info_json in queue__msg_count_info_map.items():
             info_dict = json.loads(info_json)
-            if info_dict['report_ts'] > time.time() - 15 and info_dict['last_get_msg_num_ts'] > time.time() - 1200:
+            if ignore_report_ts or (info_dict['report_ts'] > time.time() - 15 and info_dict['last_get_msg_num_ts'] > time.time() - 1200):
                 queue__msg_count_dict[queue_name] = info_dict['msg_num_in_broker']
         return queue__msg_count_dict
 
@@ -127,7 +127,7 @@ class QueueConusmerParamsGetter(RedisMixin, FunboostFileLoggerMixin):
         queue__active_consumers_map = ActiveCousumerProcessInfoGetter().get_all_hearbeat_info_partition_by_queue_name()
         queue__consumer_params_map  = self.get_queue_params()
         queue__pause_map = self.get_pause_flag()
-        queue__msg_count_dict = self.get_msg_num()
+        queue__msg_count_dict = self.get_msg_num(ignore_report_ts=True)
         queue_params_and_active_consumers = {}
 
         for queue, consumer_params in  queue__consumer_params_map.items():

funboost/core/booster.py

@@ -307,7 +307,7 @@ class BoostersManager:
         跨不同的项目，发布消息。例如proj1中定义有fun1消费函数，但proj2无法直接到日proj1的函数，无法直接 fun1.push 来发布消息
         可以使用这个方法，获取一个publisher。
 
-        publisher = BoostersManager.get_cross_project_publisher(PublisherParams(queue_name='proj1_queue', broker_kind=BrokerEnum.SQLITE_QUEUE))
+        publisher = BoostersManager.get_cross_project_publisher(PublisherParams(queue_name='proj1_queue', broker_kind=publisher_params.broker_kind))
         publisher.publish({'x': aaa})
         """
         pid = os.getpid()

funboost/core/cli/funboost_fire.py

@@ -127,3 +127,13 @@ class BoosterFire(object):
         """
         for queue_name in queue_names:
             BoostersManager.get_booster(queue_name).continue_consume()
+    
+    def start_funboost_web_manager(self):
+        """
+        启动funboost web管理器;
+        例子: start_funboost_web_manager
+        """
+        from funboost.function_result_web.app import start_funboost_web_manager
+        start_funboost_web_manager()
+
+    start_web = start_funboost_web_manager

funboost/core/msg_result_getter.py

@@ -52,6 +52,7 @@ class AsyncResult(RedisMixin):
     @property
     def status_and_result(self):
         if not self._has_pop:
+            # print(f'{self.task_id} 正在等待结果')
             redis_value = self.redis_db_filter_and_rpc_result.blpop(self.task_id, self.timeout)
             self._has_pop = True
             if redis_value is not None:

funboost/function_result_web/app.py

@@ -12,6 +12,9 @@ import os
 
 import datetime
 import json
+import traceback
+
+from funboost.core.func_params_model import PriorityConsumingControlConfig
 
 """
 pip install Flask flask_bootstrap  flask_wtf  wtforms flask_login       
@@ -26,6 +29,7 @@ from flask_login import login_user, logout_user, login_required, LoginManager, U
 import nb_log
 from funboost import nb_print,ActiveCousumerProcessInfoGetter,BoostersManager,PublisherParams,RedisMixin
 from funboost.function_result_web.functions import get_cols, query_result, get_speed, Statistic
+from funboost.function_result_web import functions as app_functions
 from funboost.core.active_cousumer_info_getter import QueueConusmerParamsGetter
 from funboost.constant import RedisKeys
 
@@ -119,7 +123,8 @@ def logout():
 @app.route('/')
 @login_required
 def index():
-    return render_template('index.html')
+    page = request.args.get('page')
+    return render_template('index.html', page=page)
 
 
 @app.route('/query_cols')
@@ -238,6 +243,39 @@ def resume_consume(queue_name):
     RedisMixin().redis_db_frame.hset(RedisKeys.REDIS_KEY_PAUSE_FLAG, queue_name,'0')
     return jsonify({'success':True})
 
+@app.route('/queue/get_msg_num',methods=['GET'])
+def get_msg_num():
+    return jsonify(QueueConusmerParamsGetter().get_msg_num(ignore_report_ts=True))
+        
+@app.route('/rpc/rpc_call',methods=['POST'])
+def rpc_call():
+    """
+    class MsgItem(BaseModel):
+        queue_name: str  # 队列名
+        msg_body: dict  # 消息体,就是boost函数的入参字典,例如 {"x":1,"y":2}
+        need_result: bool = False  # 发布消息后,是否需要返回结果
+        timeout: int = 60  # 等待结果返回的最大等待时间.
+
+
+    class PublishResponse(BaseModel):
+        succ: bool
+        msg: str
+        status_and_result: typing.Optional[dict] = None  # 消费函数的消费状态和结果.
+        task_id:str
+    """
+    
+    msg_item = request.get_json()
+    return jsonify(app_functions.rpc_call(**msg_item))
+    
+@app.route('/rpc/get_result_by_task_id',methods=['GET'])
+def get_result_by_task_id():
+    res = app_functions.get_result_by_task_id(task_id=request.args.get('task_id'),
+                                                          timeout=request.args.get('timeout') or 60)
+    if res['status_and_result'] is None:
+        return jsonify({'succ':False,'msg':'task_id不存在或者超时或者结果已经过期'})
+    return jsonify(res)
+   
+
 def start_funboost_web_manager(host='0.0.0.0', port=27018,block=False):
     print('start_funboost_web_manager , sys.path :', sys.path)
     def _start_funboost_web_manager():
@@ -246,7 +284,9 @@ def start_funboost_web_manager(host='0.0.0.0', port=27018,block=False):
         _start_funboost_web_manager()
     else:
         threading.Thread(target=_start_funboost_web_manager).start()
-
+        
+        
+        
 if __name__ == '__main__':
     # app.jinja_env.auto_reload = True
     # with app.test_request_context():

funboost/function_result_web/app_debug_start.py

@@ -0,0 +1,6 @@
+from funboost.function_result_web.app import app
+
+
+if __name__ == '__main__':
+    app.run(debug=True, threaded=True, host='0.0.0.0', port=27018)
+

funboost/function_result_web/functions.py

@@ -6,9 +6,16 @@ import json
 from pprint import pprint
 import time
 import copy
+import traceback
 from funboost import nb_print
+from funboost.constant import RedisKeys
+from funboost.core.booster import BoostersManager
+from funboost.core.func_params_model import PriorityConsumingControlConfig, PublisherParams
+from funboost.core.msg_result_getter import AsyncResult
+from funboost.core.serialization import Serialization
 from funboost.utils import time_util, decorators, LoggerMixin
 from funboost.utils.mongo_util import MongoMixin
+from funboost.utils.redis_manager import RedisMixin
 
 # from test_frame.my_patch_frame_config import do_patch_frame_config
 #
@@ -144,6 +151,46 @@ class Statistic(LoggerMixin):
                                                  t2.strftime('%Y-%m-%d %H:%M:%S'))
                 self.result['recent_60_seconds']['count_arr'].append(count)
 
+def rpc_call(queue_name, msg_body, need_result, timeout):
+  
+    status_and_result = None
+    task_id = None
+    try:
+        boost_params_json = RedisMixin().redis_db_frame.hget(RedisKeys.FUNBOOST_QUEUE__CONSUMER_PARAMS,queue_name)
+        boost_params_dict = Serialization.to_dict(boost_params_json)
+        broker_kind = boost_params_dict['broker_kind']
+        publisher = BoostersManager.get_cross_project_publisher(PublisherParams(queue_name=queue_name,
+                                                                            broker_kind=broker_kind, 
+                                                                            publish_msg_log_use_full_msg=True))
+    
+        if need_result:
+            # if booster.boost_params.is_using_rpc_mode is False:
+            #     raise ValueError(f' need_result 为true,{booster.queue_name} 队列消费者 需要@boost设置支持rpc模式')
+            
+            async_result =  publisher.publish(msg_body,priority_control_config=PriorityConsumingControlConfig(is_using_rpc_mode=True))
+            async_result.set_timeout(timeout)
+            status_and_result = async_result.status_and_result
+            # print(status_and_result)
+            task_id = async_result.task_id
+        else:
+            async_result =publisher.publish(msg_body)
+            task_id = async_result.task_id
+        return dict(succ=True, msg=f'{queue_name} 队列,消息发布成功', 
+                            status_and_result=status_and_result,task_id=task_id)
+    except Exception as e:
+        return dict(succ=False, msg=f'{queue_name} 队列,消息发布失败 {type(e)} {e} {traceback.format_exc()}',
+                               status_and_result=status_and_result,task_id=task_id)
+    
+
+def get_result_by_task_id(task_id,timeout):
+    async_result = AsyncResult(task_id)
+    async_result.set_timeout(timeout)
+    status_and_result = async_result.status_and_result
+    return dict(succ=True, msg=f'task_id:{task_id} 获取结果成功', 
+                            status_and_result=status_and_result,task_id=task_id)
+        
+    
+     
 
 if __name__ == '__main__':
     # print(get_cols('4'))
@@ -152,6 +199,13 @@ if __name__ == '__main__':
     # nb_print(get_speed_last_minute('queue_test54'))
 
     # nb_print(get_speed('queue_test56', '2019-09-18 16:03:29', '2019-09-23 16:03:29'))
-    stat = Statistic('queue_test_f01t')
-    stat.build_result()
-    nb_print(stat.result)
+    # stat = Statistic('queue_test_f01t')
+    # stat.build_result()
+    # nb_print(stat.result)
+    
+    # res = rpc_call('queue_test_g02t',{'x':1,'y':2},True,60)
+    
+    res = get_result_by_task_id('3232',60)
+    print(res)
+    
+