funboost 44.8__py3-none-any.whl → 45.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of funboost might be problematic. Click here for more details.
- funboost/__init__.py +1 -1
- funboost/constant.py +14 -4
- funboost/consumers/base_consumer.py +48 -20
- funboost/core/booster.py +15 -0
- funboost/core/current_task.py +88 -69
- funboost/core/func_params_model.py +8 -0
- funboost/function_result_web/__pycache__/functions.cpython-39.pyc +0 -0
- funboost/publishers/base_publisher.py +21 -2
- funboost/set_frame_config.py +2 -1
- funboost/utils/class_utils.py +101 -51
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/add_to_pythonpath.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__init__.py +59 -59
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/__init__.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/client.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/compat.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/connection.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/exceptions.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/lock.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/utils.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/client.py +4804 -4804
- funboost/utils/dependency_packages_in_pythonpath/aioredis/compat.py +8 -8
- funboost/utils/dependency_packages_in_pythonpath/aioredis/connection.py +1668 -1668
- funboost/utils/dependency_packages_in_pythonpath/aioredis/exceptions.py +96 -96
- funboost/utils/dependency_packages_in_pythonpath/aioredis/lock.py +306 -306
- funboost/utils/dependency_packages_in_pythonpath/aioredis/log.py +15 -15
- funboost/utils/dependency_packages_in_pythonpath/aioredis/sentinel.py +329 -329
- funboost/utils/dependency_packages_in_pythonpath/aioredis/utils.py +61 -61
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/StoppableThread.py +134 -133
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__init__.py +16 -16
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/StoppableThread.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/__init__.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/dafunc.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/exceptions.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-39.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/dafunc.py +244 -244
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/dafunc2222.py +244 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/exceptions.py +98 -98
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/py2_raise.py +7 -7
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/py3_raise.py +7 -7
- funboost/utils/times/__init__.py +85 -85
- funboost/utils/times/version.py +1 -1
- {funboost-44.8.dist-info → funboost-45.0.dist-info}/METADATA +1 -2
- {funboost-44.8.dist-info → funboost-45.0.dist-info}/RECORD +48 -76
- {funboost-44.8.dist-info → funboost-45.0.dist-info}/entry_points.txt +0 -1
- funboost/function_result_web/__pycache__/app.cpython-37.pyc +0 -0
- funboost/function_result_web/__pycache__/functions.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/__init__.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/add_to_pythonpath.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/__pycache__/add_to_pythonpath.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/__init__.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/__init__.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/client.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/client.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/compat.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/compat.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/connection.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/connection.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/exceptions.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/exceptions.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/lock.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/lock.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/utils.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/aioredis/__pycache__/utils.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/StoppableThread.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/StoppableThread.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/__init__.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/__init__.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/dafunc.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/dafunc.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/exceptions.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/exceptions.cpython-37.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-311.pyc +0 -0
- funboost/utils/dependency_packages_in_pythonpath/func_timeout/__pycache__/py3_raise.cpython-37.pyc +0 -0
- {funboost-44.8.dist-info → funboost-45.0.dist-info}/LICENSE +0 -0
- {funboost-44.8.dist-info → funboost-45.0.dist-info}/WHEEL +0 -0
- {funboost-44.8.dist-info → funboost-45.0.dist-info}/top_level.txt +0 -0
funboost/__init__.py
CHANGED
funboost/constant.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# coding= utf-8
|
|
2
2
|
class BrokerEnum:
|
|
3
|
-
EMPTY = 'empty'
|
|
3
|
+
EMPTY = 'empty' # 空的实现,需要搭配 boost入参的 consumer_override_cls 和 publisher_override_cls使用,或者被继承。
|
|
4
4
|
|
|
5
5
|
RABBITMQ_AMQPSTORM = 'RABBITMQ_AMQPSTORM' # 使用 amqpstorm 包操作rabbitmq 作为 分布式消息队列,支持消费确认.强烈推荐这个作为funboost中间件。
|
|
6
6
|
RABBITMQ = RABBITMQ_AMQPSTORM
|
|
@@ -41,8 +41,6 @@ class BrokerEnum:
|
|
|
41
41
|
|
|
42
42
|
ZEROMQ = 'ZEROMQ' # 基于zeromq作为分布式消息队列,不需要安装中间件,可以支持跨机器但不支持持久化。
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
|
|
46
44
|
"""
|
|
47
45
|
操作 kombu 包,这个包也是celery的中间件依赖包,这个包可以操作10种中间件(例如rabbitmq redis),但没包括分布式函数调度框架的kafka nsq zeromq 等。
|
|
48
46
|
同时 kombu 包的性能非常差,可以用原生redis的lpush和kombu的publish测试发布,使用brpop 和 kombu 的 drain_events测试消费,对比差距相差了5到10倍。
|
|
@@ -71,7 +69,6 @@ class BrokerEnum:
|
|
|
71
69
|
|
|
72
70
|
PEEWEE = 'PEEWEE' # peewee包操作mysql,使用表模拟消息队列
|
|
73
71
|
|
|
74
|
-
|
|
75
72
|
CELERY = 'CELERY' # funboost支持celery框架来发布和消费任务,由celery框架来调度执行任务,但是写法简单远远暴击用户亲自使用celery的麻烦程度,
|
|
76
73
|
# 用户永无无需关心和操作Celery对象实例,无需关心celery的task_routes和include配置,funboost来自动化设置这些celery配置。
|
|
77
74
|
|
|
@@ -92,4 +89,17 @@ class ConcurrentModeEnum:
|
|
|
92
89
|
SINGLE_THREAD = 'single_thread' # 如果你不想并发,不想预先从消息队列中间件拉取消息到python程序的内存queue队列缓冲中,那么就适合使用此并发模式。
|
|
93
90
|
SOLO = SINGLE_THREAD
|
|
94
91
|
|
|
92
|
+
|
|
95
93
|
# is_fsdf_remote_run = 0
|
|
94
|
+
|
|
95
|
+
class FunctionKind:
|
|
96
|
+
CLASS_METHOD = 'CLASS_METHOD'
|
|
97
|
+
INSTANCE_METHOD = 'INSTANCE_METHOD'
|
|
98
|
+
STATIC_METHOD = 'STATIC_METHOD'
|
|
99
|
+
COMMON_FUNCTION = 'COMMON_FUNCTION'
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class ConstStrForClassMethod:
|
|
103
|
+
FIRST_PARAM_NAME = 'first_param_name'
|
|
104
|
+
CLS_NAME = 'cls_name'
|
|
105
|
+
OBJ_INIT_PARAMS = 'obj_init_params'
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
框架做主要的功能都是在这个文件里面实现的.
|
|
9
9
|
"""
|
|
10
10
|
import functools
|
|
11
|
+
import sys
|
|
11
12
|
import typing
|
|
12
13
|
import abc
|
|
13
14
|
import copy
|
|
@@ -30,13 +31,14 @@ from threading import Lock
|
|
|
30
31
|
import asyncio
|
|
31
32
|
|
|
32
33
|
import nb_log
|
|
33
|
-
from funboost.core.current_task import funboost_current_task
|
|
34
|
+
from funboost.core.current_task import funboost_current_task, FctContext
|
|
34
35
|
from funboost.core.loggers import develop_logger
|
|
35
36
|
|
|
36
37
|
from funboost.core.func_params_model import BoosterParams, PublisherParams, BaseJsonAbleModel
|
|
37
38
|
from funboost.core.task_id_logger import TaskIdLogger
|
|
39
|
+
from funboost.constant import FunctionKind
|
|
38
40
|
from funboost.utils.json_helper import JsonUtils
|
|
39
|
-
from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager,
|
|
41
|
+
from nb_log import (get_logger, LoggerLevelSetterMixin, LogManager, is_main_process,
|
|
40
42
|
nb_log_config_default)
|
|
41
43
|
from funboost.core.loggers import FunboostFileLoggerMixin, logger_prompt
|
|
42
44
|
|
|
@@ -66,7 +68,7 @@ from funboost.consumers.redis_filter import RedisFilter, RedisImpermanencyFilter
|
|
|
66
68
|
from funboost.factories.publisher_factotry import get_publisher
|
|
67
69
|
|
|
68
70
|
from funboost.utils import decorators, time_util, redis_manager
|
|
69
|
-
from funboost.constant import ConcurrentModeEnum, BrokerEnum
|
|
71
|
+
from funboost.constant import ConcurrentModeEnum, BrokerEnum, ConstStrForClassMethod
|
|
70
72
|
from funboost.core import kill_remote_task
|
|
71
73
|
from funboost.core.exceptions import ExceptionForRequeue, ExceptionForPushToDlxqueue
|
|
72
74
|
|
|
@@ -562,12 +564,35 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
562
564
|
"""
|
|
563
565
|
self._do_not_delete_extra_from_msg = True
|
|
564
566
|
|
|
565
|
-
def user_custom_record_process_info_func(self,current_function_result_status:FunctionResultStatus): # 这个可以继承
|
|
567
|
+
def user_custom_record_process_info_func(self, current_function_result_status: FunctionResultStatus): # 这个可以继承
|
|
566
568
|
pass
|
|
567
569
|
|
|
568
|
-
async def aio_user_custom_record_process_info_func(self,current_function_result_status:FunctionResultStatus): # 这个可以继承
|
|
570
|
+
async def aio_user_custom_record_process_info_func(self, current_function_result_status: FunctionResultStatus): # 这个可以继承
|
|
569
571
|
pass
|
|
570
572
|
|
|
573
|
+
def _convert_real_function_only_params_by_conusuming_function_kind(self, function_only_params: dict):
|
|
574
|
+
"""对于实例方法和classmethod 方法, 从消息队列的消息恢复第一个入参, self 和 cls"""
|
|
575
|
+
if self.consumer_params.consuming_function_kind in [FunctionKind.CLASS_METHOD, FunctionKind.INSTANCE_METHOD]:
|
|
576
|
+
real_function_only_params = copy.copy(function_only_params)
|
|
577
|
+
method_first_param_name = None
|
|
578
|
+
method_first_param_value = None
|
|
579
|
+
for k, v in function_only_params.items():
|
|
580
|
+
if isinstance(v, dict) and ConstStrForClassMethod.FIRST_PARAM_NAME in v:
|
|
581
|
+
method_first_param_name = k
|
|
582
|
+
method_first_param_value = v
|
|
583
|
+
break
|
|
584
|
+
method_cls = getattr(sys.modules[self.consumer_params.consuming_function_class_module],
|
|
585
|
+
self.consumer_params.consuming_function_class_name)
|
|
586
|
+
if self.publisher_params.consuming_function_kind == FunctionKind.CLASS_METHOD:
|
|
587
|
+
real_function_only_params[method_first_param_name] = method_cls
|
|
588
|
+
elif self.publisher_params.consuming_function_kind == FunctionKind.INSTANCE_METHOD:
|
|
589
|
+
obj = method_cls(**method_first_param_value[ConstStrForClassMethod.OBJ_INIT_PARAMS])
|
|
590
|
+
real_function_only_params[method_first_param_name] = obj
|
|
591
|
+
# print(real_function_only_params)
|
|
592
|
+
return real_function_only_params
|
|
593
|
+
else:
|
|
594
|
+
return function_only_params
|
|
595
|
+
|
|
571
596
|
# noinspection PyProtectedMember
|
|
572
597
|
def _run(self, kw: dict, ):
|
|
573
598
|
# print(kw)
|
|
@@ -627,9 +652,9 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
627
652
|
msg = f'{self._unit_time_for_count} 秒内执行了 {self._execute_task_times_every_unit_time} 次函数 [ {self.consuming_function.__name__} ] ,' \
|
|
628
653
|
f'函数平均运行耗时 {avarage_function_spend_time} 秒。 '
|
|
629
654
|
self.logger.info(msg)
|
|
630
|
-
if
|
|
655
|
+
if time.time() - self._last_show_remaining_execution_time > self._show_remaining_execution_time_interval:
|
|
631
656
|
self._msg_num_in_broker = self.publisher_of_same_queue.get_message_count()
|
|
632
|
-
if self._msg_num_in_broker != -1
|
|
657
|
+
if self._msg_num_in_broker != -1: # 有的中间件无法统计或没实现统计队列剩余数量的,统一返回的是-1,不显示这句话。
|
|
633
658
|
# msg += f''' ,预计还需要 {time_util.seconds_to_hour_minute_second(self._msg_num_in_broker * avarage_function_spend_time / active_consumer_num)} 时间 才能执行完成 {self._msg_num_in_broker}个剩余的任务'''
|
|
634
659
|
need_time = time_util.seconds_to_hour_minute_second(self._msg_num_in_broker / (self._execute_task_times_every_unit_time / self._unit_time_for_count) /
|
|
635
660
|
self._distributed_consumer_statistics.active_consumer_num)
|
|
@@ -639,7 +664,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
639
664
|
self._current_time_for_execute_task_times_every_unit_time = time.time()
|
|
640
665
|
self._consuming_function_cost_time_total_every_unit_time = 0
|
|
641
666
|
self._execute_task_times_every_unit_time = 0
|
|
642
|
-
self.user_custom_record_process_info_func(current_function_result_status)
|
|
667
|
+
self.user_custom_record_process_info_func(current_function_result_status) # 两种方式都可以自定义,记录结果,建议继承方式,不使用boost中指定 user_custom_record_process_info_func
|
|
643
668
|
if self.consumer_params.user_custom_record_process_info_func:
|
|
644
669
|
self.consumer_params.user_custom_record_process_info_func(current_function_result_status)
|
|
645
670
|
except BaseException as e:
|
|
@@ -656,17 +681,19 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
656
681
|
t_start = time.time()
|
|
657
682
|
# function_result_status.run_times = current_retry_times + 1
|
|
658
683
|
fct = funboost_current_task()
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
684
|
+
fct_context = FctContext(function_params=function_only_params,
|
|
685
|
+
full_msg=kw['body'],
|
|
686
|
+
function_result_status=function_result_status,
|
|
687
|
+
logger=self.logger, )
|
|
688
|
+
|
|
663
689
|
try:
|
|
664
690
|
function_run = self.consuming_function
|
|
665
691
|
if self._consuming_function_is_asyncio:
|
|
666
|
-
|
|
692
|
+
fct_context.asyncio_use_thread_concurrent_mode = True
|
|
667
693
|
function_run = sync_or_async_fun_deco(function_run)
|
|
668
694
|
else:
|
|
669
|
-
|
|
695
|
+
fct_context.asynco_use_thread_concurrent_mode = False
|
|
696
|
+
fct.set_fct_context(fct_context)
|
|
670
697
|
function_timeout = self._get_priority_conf(kw, 'function_timeout')
|
|
671
698
|
function_run = function_run if self.consumer_params.consumin_function_decorator is None else self.consumer_params.consumin_function_decorator(function_run)
|
|
672
699
|
function_run = function_run if not function_timeout else self._concurrent_mode_dispatcher.timeout_deco(
|
|
@@ -678,7 +705,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
678
705
|
self.logger.warning(f'取消运行 {task_id} {function_only_params}')
|
|
679
706
|
return function_result_status
|
|
680
707
|
function_run = kill_remote_task.kill_fun_deco(task_id)(function_run) # 用杀死装饰器包装起来在另一个线程运行函数,以便等待远程杀死。
|
|
681
|
-
function_result_status.result = function_run(**function_only_params)
|
|
708
|
+
function_result_status.result = function_run(**self._convert_real_function_only_params_by_conusuming_function_kind(function_only_params))
|
|
682
709
|
# if asyncio.iscoroutine(function_result_status.result):
|
|
683
710
|
# log_msg = f'''异步的协程消费函数必须使用 async 并发模式并发,请设置消费函数 {self.consuming_function.__name__} 的concurrent_mode 为 ConcurrentModeEnum.ASYNC 或 4'''
|
|
684
711
|
# # self.logger.critical(msg=f'{log_msg} \n')
|
|
@@ -828,12 +855,13 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
828
855
|
# noinspection PyBroadException
|
|
829
856
|
t_start = time.time()
|
|
830
857
|
fct = funboost_current_task()
|
|
831
|
-
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
858
|
+
fct_context = FctContext(function_params=function_only_params,
|
|
859
|
+
full_msg=kw['body'],
|
|
860
|
+
function_result_status=function_result_status,
|
|
861
|
+
logger=self.logger, )
|
|
862
|
+
fct.set_fct_context(fct_context)
|
|
835
863
|
try:
|
|
836
|
-
corotinue_obj = self.consuming_function(**function_only_params)
|
|
864
|
+
corotinue_obj = self.consuming_function(**self._convert_real_function_only_params_by_conusuming_function_kind(function_only_params))
|
|
837
865
|
if not asyncio.iscoroutine(corotinue_obj):
|
|
838
866
|
log_msg = f'''当前设置的并发模式为 async 并发模式,但消费函数不是异步协程函数,请不要把消费函数 {self.consuming_function.__name__} 的 concurrent_mode 设置错误'''
|
|
839
867
|
# self.logger.critical(msg=f'{log_msg} \n')
|
funboost/core/booster.py
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
import copy
|
|
3
|
+
import inspect
|
|
3
4
|
import os
|
|
5
|
+
import sys
|
|
4
6
|
import types
|
|
5
7
|
import typing
|
|
6
8
|
|
|
7
9
|
from funboost.concurrent_pool import FlexibleThreadPool
|
|
8
10
|
from funboost.concurrent_pool.async_helper import simple_run_in_executor
|
|
11
|
+
from funboost.constant import FunctionKind
|
|
12
|
+
from funboost.utils.class_utils import ClsHelper
|
|
13
|
+
|
|
9
14
|
from funboost.utils.ctrl_c_end import ctrl_c_recv
|
|
10
15
|
from funboost.core.loggers import flogger, develop_logger, logger_prompt
|
|
11
16
|
|
|
@@ -85,6 +90,16 @@ class Booster:
|
|
|
85
90
|
if len(kwargs) == 0 and len(args) == 1 and isinstance(args[0], typing.Callable):
|
|
86
91
|
consuming_function = args[0]
|
|
87
92
|
self.boost_params.consuming_function = consuming_function
|
|
93
|
+
# print(consuming_function)
|
|
94
|
+
# print(ClsHelper.get_method_kind(consuming_function))
|
|
95
|
+
# print(inspect.getsourcelines(consuming_function))
|
|
96
|
+
if self.boost_params.consuming_function_kind is None:
|
|
97
|
+
self.boost_params.consuming_function_kind = ClsHelper.get_method_kind(consuming_function)
|
|
98
|
+
if self.boost_params.consuming_function_kind in [FunctionKind.CLASS_METHOD,FunctionKind.INSTANCE_METHOD]:
|
|
99
|
+
if self.boost_params.consuming_function_class_module is None:
|
|
100
|
+
self.boost_params.consuming_function_class_module = consuming_function.__module__
|
|
101
|
+
if self.boost_params.consuming_function_class_name is None:
|
|
102
|
+
self.boost_params.consuming_function_class_name = consuming_function.__qualname__.split('.')[0]
|
|
88
103
|
logger_prompt.debug(f''' {self.boost_params.queue_name} booster 配置是 {self.boost_params.json_str_value()}''')
|
|
89
104
|
self.consuming_function = consuming_function
|
|
90
105
|
self.is_decorated_as_consume_function = True
|
funboost/core/current_task.py
CHANGED
|
@@ -1,7 +1,10 @@
|
|
|
1
|
+
import abc
|
|
1
2
|
import contextvars
|
|
3
|
+
from dataclasses import dataclass
|
|
2
4
|
import logging
|
|
3
5
|
import threading
|
|
4
6
|
import asyncio
|
|
7
|
+
|
|
5
8
|
from funboost.core.function_result_status_saver import FunctionResultStatus
|
|
6
9
|
|
|
7
10
|
""" 用法例子
|
|
@@ -47,38 +50,55 @@ if __name__ == '__main__':
|
|
|
47
50
|
"""
|
|
48
51
|
|
|
49
52
|
|
|
50
|
-
|
|
53
|
+
@dataclass
|
|
54
|
+
class FctContext:
|
|
51
55
|
"""
|
|
52
|
-
|
|
56
|
+
fct 是 funboost current task 的简写
|
|
53
57
|
"""
|
|
54
58
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
59
|
+
function_params: dict
|
|
60
|
+
full_msg: dict
|
|
61
|
+
function_result_status: FunctionResultStatus
|
|
62
|
+
logger: logging.Logger
|
|
63
|
+
asyncio_use_thread_concurrent_mode: bool = False
|
|
64
|
+
|
|
65
|
+
# class FctContext:
|
|
66
|
+
# """
|
|
67
|
+
# fct 是 funboost current task 的简写
|
|
68
|
+
# """
|
|
69
|
+
#
|
|
70
|
+
# def __init__(self, function_params: dict,
|
|
71
|
+
# full_msg: dict,
|
|
72
|
+
# function_result_status: FunctionResultStatus,
|
|
73
|
+
# logger: logging.Logger,
|
|
74
|
+
# asyncio_use_thread_concurrent_mode: bool = False):
|
|
75
|
+
# self.function_params = function_params
|
|
76
|
+
# self.full_msg = full_msg
|
|
77
|
+
# self.function_result_status = function_result_status
|
|
78
|
+
# self.logger = logger
|
|
79
|
+
# self.asyncio_use_thread_concurrent_mode = asyncio_use_thread_concurrent_mode
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _BaseCurrentTask(metaclass=abc.ABCMeta):
|
|
83
|
+
@abc.abstractmethod
|
|
84
|
+
def set_fct_context(self, fct_context: FctContext):
|
|
85
|
+
raise NotImplemented
|
|
86
|
+
|
|
87
|
+
@abc.abstractmethod
|
|
88
|
+
def get_fct_context(self) -> FctContext:
|
|
89
|
+
raise NotImplemented
|
|
58
90
|
|
|
59
91
|
@property
|
|
60
92
|
def function_params(self):
|
|
61
|
-
return self.
|
|
62
|
-
|
|
63
|
-
@function_params.setter
|
|
64
|
-
def function_params(self, function_params: dict):
|
|
65
|
-
self._fct_local_data.function_params = function_params
|
|
93
|
+
return self.get_fct_context().function_params
|
|
66
94
|
|
|
67
95
|
@property
|
|
68
96
|
def full_msg(self) -> dict:
|
|
69
|
-
return self.
|
|
70
|
-
|
|
71
|
-
@full_msg.setter
|
|
72
|
-
def full_msg(self, full_msg: dict):
|
|
73
|
-
self._fct_local_data.full_msg = full_msg
|
|
97
|
+
return self.get_fct_context().full_msg
|
|
74
98
|
|
|
75
99
|
@property
|
|
76
100
|
def function_result_status(self) -> FunctionResultStatus:
|
|
77
|
-
return self.
|
|
78
|
-
|
|
79
|
-
@function_result_status.setter
|
|
80
|
-
def function_result_status(self, function_result_status: FunctionResultStatus):
|
|
81
|
-
self._fct_local_data.function_result_status = function_result_status
|
|
101
|
+
return self.get_fct_context().function_result_status
|
|
82
102
|
|
|
83
103
|
@property
|
|
84
104
|
def task_id(self) -> FunctionResultStatus:
|
|
@@ -86,68 +106,51 @@ class __ThreadCurrentTask:
|
|
|
86
106
|
|
|
87
107
|
@property
|
|
88
108
|
def logger(self) -> logging.Logger:
|
|
89
|
-
return self.
|
|
109
|
+
return self.get_fct_context().logger
|
|
90
110
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
self._fct_local_data.logger = logger
|
|
111
|
+
def __str__(self):
|
|
112
|
+
return f'<{self.__class__.__name__} [{self.function_result_status.get_status_dict()}]>'
|
|
94
113
|
|
|
95
|
-
thread_current_task = __ThreadCurrentTask()
|
|
96
|
-
def is_asyncio_environment():
|
|
97
|
-
try:
|
|
98
|
-
loop = asyncio.get_running_loop()
|
|
99
|
-
return True
|
|
100
|
-
except RuntimeError as e:
|
|
101
|
-
return False
|
|
102
114
|
|
|
115
|
+
class __ThreadCurrentTask(_BaseCurrentTask):
|
|
116
|
+
"""
|
|
117
|
+
用于在用户自己函数内部去获取 消息的完整体,当前重试次数等.
|
|
118
|
+
"""
|
|
103
119
|
|
|
104
|
-
|
|
105
|
-
_function_params = contextvars.ContextVar("function_params")
|
|
106
|
-
_full_msg = contextvars.ContextVar("full_msg")
|
|
107
|
-
_function_result_status = contextvars.ContextVar("function_result_status")
|
|
108
|
-
_logger = contextvars.ContextVar('logger')
|
|
120
|
+
_fct_local_data = threading.local()
|
|
109
121
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
return self._function_params.get()
|
|
122
|
+
def set_fct_context(self, fct_context: FctContext):
|
|
123
|
+
self._fct_local_data.fct_context = fct_context
|
|
113
124
|
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
self._function_params.set(function_params)
|
|
125
|
+
def get_fct_context(self) -> FctContext:
|
|
126
|
+
return self._fct_local_data.fct_context
|
|
117
127
|
|
|
118
|
-
@property
|
|
119
|
-
def full_msg(self) -> dict:
|
|
120
|
-
return self._full_msg.get()
|
|
121
128
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
self._full_msg.set(full_msg)
|
|
129
|
+
class __AsyncioCurrentTask(_BaseCurrentTask):
|
|
130
|
+
_fct_context = contextvars.ContextVar('fct_context')
|
|
125
131
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
return self._function_result_status.get()
|
|
132
|
+
def set_fct_context(self, fct_context: FctContext):
|
|
133
|
+
self._fct_context.set(fct_context)
|
|
129
134
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
self._function_result_status.set(function_result_status)
|
|
135
|
+
def get_fct_context(self) -> FctContext:
|
|
136
|
+
return self._fct_context.get()
|
|
133
137
|
|
|
134
|
-
@property
|
|
135
|
-
def task_id(self) :
|
|
136
|
-
return self.function_result_status.task_id
|
|
137
138
|
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
return self._logger.get()
|
|
139
|
+
thread_current_task = __ThreadCurrentTask()
|
|
140
|
+
asyncio_current_task = __AsyncioCurrentTask()
|
|
141
141
|
|
|
142
|
-
@logger.setter
|
|
143
|
-
def logger(self, logger: logging.Logger):
|
|
144
|
-
self._logger.set(logger)
|
|
145
142
|
|
|
146
|
-
|
|
143
|
+
def is_asyncio_environment():
|
|
144
|
+
try:
|
|
145
|
+
asyncio.get_running_loop()
|
|
146
|
+
return True
|
|
147
|
+
except RuntimeError:
|
|
148
|
+
return False
|
|
149
|
+
|
|
147
150
|
|
|
148
151
|
def funboost_current_task():
|
|
149
152
|
if is_asyncio_environment():
|
|
150
|
-
if
|
|
153
|
+
if thread_current_task.get_fct_context().asyncio_use_thread_concurrent_mode is True:
|
|
151
154
|
# 如果用户使用的是默认的ConcurrentModeEnum.THREADING并发模式来运行async def 函数,那么也使用线程获取上下文
|
|
152
155
|
return thread_current_task
|
|
153
156
|
else:
|
|
@@ -157,18 +160,34 @@ def funboost_current_task():
|
|
|
157
160
|
|
|
158
161
|
|
|
159
162
|
def get_current_taskid():
|
|
160
|
-
fct = funboost_current_task()
|
|
161
163
|
# return fct.function_result_status.task_id
|
|
162
164
|
try:
|
|
165
|
+
fct = funboost_current_task()
|
|
163
166
|
return fct.task_id # 不在funboost的消费函数里面就获取不到上下文了
|
|
164
|
-
except (AttributeError,LookupError) as e:
|
|
167
|
+
except (AttributeError, LookupError) as e:
|
|
165
168
|
# print(e,type(e))
|
|
166
169
|
return 'no_task_id'
|
|
167
170
|
|
|
171
|
+
|
|
172
|
+
class FctContextThread(threading.Thread):
|
|
173
|
+
"""
|
|
174
|
+
这个类自动把当前线程的 线程上下文 自动传递给新开的线程。
|
|
175
|
+
"""
|
|
176
|
+
def __init__(self, group=None, target=None, name=None,
|
|
177
|
+
args=(), kwargs=None, *, daemon=None,
|
|
178
|
+
):
|
|
179
|
+
threading.Thread.__init__(**locals())
|
|
180
|
+
self.fct_context = thread_current_task.get_fct_context()
|
|
181
|
+
|
|
182
|
+
def run(self):
|
|
183
|
+
thread_current_task.set_fct_context(self.fct_context)
|
|
184
|
+
super().run()
|
|
185
|
+
|
|
186
|
+
|
|
168
187
|
if __name__ == '__main__':
|
|
169
188
|
print(is_asyncio_environment())
|
|
170
189
|
print()
|
|
171
|
-
for i in range(
|
|
190
|
+
for i in range(2):
|
|
172
191
|
funboost_current_task()
|
|
173
|
-
get_current_taskid()
|
|
192
|
+
print(get_current_taskid())
|
|
174
193
|
print()
|
|
@@ -193,6 +193,10 @@ class BoosterParams(BaseJsonAbleModel):
|
|
|
193
193
|
# func_params_is_pydantic_model: bool = False # funboost 兼容支持 函数娼还是 pydantic model类型,funboost在发布之前和取出来时候自己转化。
|
|
194
194
|
|
|
195
195
|
auto_generate_info: dict = {} # 自动生成的信息,不需要用户主动传参.
|
|
196
|
+
consuming_function_class:typing.Optional[typing.Type] = None
|
|
197
|
+
consuming_function_kind :typing.Optional[str]= None #自动生成的信息,不需要用户主动传参.
|
|
198
|
+
consuming_function_class_module:typing.Optional[str] = None #自动生成的信息,不需要用户主动传参.
|
|
199
|
+
consuming_function_class_name: typing.Optional[str] = None #自动生成的信息,不需要用户主动传参.
|
|
196
200
|
|
|
197
201
|
@root_validator(skip_on_failure=True)
|
|
198
202
|
def check_values(cls, values: dict):
|
|
@@ -290,6 +294,10 @@ class PublisherParams(BaseJsonAbleModel):
|
|
|
290
294
|
publisher_override_cls: typing.Optional[typing.Type] = None
|
|
291
295
|
# func_params_is_pydantic_model: bool = False # funboost 兼容支持 函数娼还是 pydantic model类型,funboost在发布之前和取出来时候自己转化。
|
|
292
296
|
|
|
297
|
+
consuming_function_kind: typing.Optional[str] = None # 自动生成的信息,不需要用户主动传参.
|
|
298
|
+
consuming_function_class_module: typing.Optional[str] = None # 自动生成的信息,不需要用户主动传参.
|
|
299
|
+
consuming_function_class_name: typing.Optional[str] = None # 自动生成的信息,不需要用户主动传参.
|
|
300
|
+
|
|
293
301
|
|
|
294
302
|
if __name__ == '__main__':
|
|
295
303
|
from funboost.concurrent_pool import FlexibleThreadPool
|
|
Binary file
|
|
@@ -16,6 +16,7 @@ from threading import Lock
|
|
|
16
16
|
import amqpstorm
|
|
17
17
|
|
|
18
18
|
import nb_log
|
|
19
|
+
from funboost.constant import ConstStrForClassMethod, FunctionKind
|
|
19
20
|
from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
|
|
20
21
|
from funboost.core.helper_funs import MsgGenerater
|
|
21
22
|
from funboost.core.loggers import develop_logger
|
|
@@ -29,6 +30,8 @@ from funboost.core.task_id_logger import TaskIdLogger
|
|
|
29
30
|
from funboost.utils import decorators
|
|
30
31
|
from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
|
|
31
32
|
|
|
33
|
+
|
|
34
|
+
|
|
32
35
|
RedisAsyncResult = AsyncResult # 别名
|
|
33
36
|
RedisAioAsyncResult = AioAsyncResult # 别名
|
|
34
37
|
|
|
@@ -243,15 +246,31 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
|
|
|
243
246
|
:param func_kwargs:
|
|
244
247
|
:return:
|
|
245
248
|
"""
|
|
246
|
-
# print(func_args,func_kwargs,self.publish_params_checker.all_arg_name)
|
|
249
|
+
# print(func_args, func_kwargs, self.publish_params_checker.all_arg_name)
|
|
247
250
|
msg_dict = func_kwargs
|
|
248
251
|
# print(msg_dict)
|
|
249
252
|
# print(self.publish_params_checker.position_arg_name_list)
|
|
250
253
|
# print(func_args)
|
|
251
|
-
|
|
254
|
+
func_args_list = list(func_args)
|
|
255
|
+
if self.publisher_params.consuming_function_kind == FunctionKind.CLASS_METHOD:
|
|
256
|
+
# print(self.publish_params_checker.all_arg_name[0])
|
|
257
|
+
# func_args_list.insert(0, {'first_param_name': self.publish_params_checker.all_arg_name[0],
|
|
258
|
+
# 'cls_type': ClsHelper.get_classs_method_cls(self.publisher_params.consuming_function).__name__},
|
|
259
|
+
# )
|
|
260
|
+
func_args_list.insert(0, {ConstStrForClassMethod.FIRST_PARAM_NAME: self.publish_params_checker.all_arg_name[0],
|
|
261
|
+
ConstStrForClassMethod.CLS_NAME: self.publisher_params.consuming_function_class_name,})
|
|
262
|
+
elif self.publisher_params.consuming_function_kind == FunctionKind.INSTANCE_METHOD:
|
|
263
|
+
if not hasattr(func_args[0],ConstStrForClassMethod.OBJ_INIT_PARAMS):
|
|
264
|
+
raise ValueError(f'消费函数 {self.publisher_params.consuming_function} 是实例方法,实例必须有 {ConstStrForClassMethod.OBJ_INIT_PARAMS} 属性')
|
|
265
|
+
func_args_list[0] = {ConstStrForClassMethod.FIRST_PARAM_NAME: self.publish_params_checker.all_arg_name[0],
|
|
266
|
+
ConstStrForClassMethod.OBJ_INIT_PARAMS: getattr(func_args[0],ConstStrForClassMethod.OBJ_INIT_PARAMS),
|
|
267
|
+
ConstStrForClassMethod.CLS_NAME: self.publisher_params.consuming_function_class_name}
|
|
268
|
+
|
|
269
|
+
for index, arg in enumerate(func_args_list):
|
|
252
270
|
# print(index,arg,self.publish_params_checker.position_arg_name_list)
|
|
253
271
|
# msg_dict[self.publish_params_checker.position_arg_name_list[index]] = arg
|
|
254
272
|
msg_dict[self.publish_params_checker.all_arg_name[index]] = arg
|
|
273
|
+
|
|
255
274
|
# print(msg_dict)
|
|
256
275
|
return self.publish(msg_dict)
|
|
257
276
|
|
funboost/set_frame_config.py
CHANGED
|
@@ -100,7 +100,7 @@ def show_frame_config():
|
|
|
100
100
|
# only_print_on_main_process(f'{var_name}: {var_value}')
|
|
101
101
|
logger_prompt.debug(f'''读取的 BrokerConnConfig 配置是:\n {funboost_config_deafult.BrokerConnConfig().get_pwd_enc_json(indent=4)} ''')
|
|
102
102
|
|
|
103
|
-
logger_prompt.debug(f'''读取的 FunboostCommonConfig 配置是:\n {funboost_config_deafult.FunboostCommonConfig().get_json(indent=
|
|
103
|
+
logger_prompt.debug(f'''读取的 FunboostCommonConfig 配置是:\n {funboost_config_deafult.FunboostCommonConfig().get_json(indent=4)} ''')
|
|
104
104
|
|
|
105
105
|
# only_print_on_main_process(f'读取的 BoostDecoratorDefaultParams 默认 @boost 装饰器入参的默认全局配置是: \n '
|
|
106
106
|
# f'{funboost_config_deafult.BoostDecoratorDefaultParams().get_json()}')
|
|
@@ -118,6 +118,7 @@ def use_config_form_funboost_config_module():
|
|
|
118
118
|
当第一次运行脚本时候,函数调度框架会在你的python当前项目的根目录下 {project_root_path} 下,创建一个名为 funboost_config.py 的文件。
|
|
119
119
|
自动读取配置,会优先读取启动脚本的所在目录 {current_script_path} 的funboost_config.py文件,
|
|
120
120
|
如果没有 {current_script_path}/funboost_config.py 文件,则读取项目根目录 {project_root_path} 下的funboost_config.py做配置。
|
|
121
|
+
只要 funboost_config.py 在任意 PYTHONPATH 的文件夹下,就能自动读取到。
|
|
121
122
|
在 "{project_root_path}/funboost_config.py:1" 文件中,需要按需重新设置要使用到的中间件的键和值,例如没有使用rabbitmq而是使用redis做中间件,则不需要配置rabbitmq。
|
|
122
123
|
"""
|
|
123
124
|
# sys.stdout.write(f'\033[0;33m{time.strftime("%H:%M:%S")}\033[0m "{__file__}:{sys._getframe().f_lineno}" \033[0;30;43m{inspect_msg}\033[0m\n')
|