funboost 43.0__tar.gz → 43.2__tar.gz

{funboost-43.0/funboost.egg-info → funboost-43.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.0
+Version: 43.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-43.0 → funboost-43.2}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "43.0"
+__version__ = "43.2"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-43.0 → funboost-43.2}/funboost/constant.py

@@ -89,5 +89,6 @@ class ConcurrentModeEnum:
     EVENTLET = 'eventlet'
     ASYNC = 'async'  # asyncio并发，适用于async def定义的函数。
     SINGLE_THREAD = 'single_thread'  # 如果你不想并发，不想预先从消息队列中间件拉取消息到python程序的内存queue队列缓冲中，那么就适合使用此并发模式。
+    SOLO = SINGLE_THREAD
 
 # is_fsdf_remote_run = 0

{funboost-43.0 → funboost-43.2}/funboost/consumers/base_consumer.py

@@ -69,8 +69,9 @@ from funboost.utils import decorators, time_util, redis_manager
 from funboost.constant import ConcurrentModeEnum, BrokerEnum
 from funboost.core import kill_remote_task
 from funboost.core.exceptions import ExceptionForRequeue, ExceptionForPushToDlxqueue
+
 # from funboost.core.booster import BoostersManager  互相导入
-from funboost.core.lazy_impoter import LazyImpoter
+from funboost.core.lazy_impoter import lazy_impoter
 
 
 # patch_apscheduler_run_job()
@@ -219,6 +220,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                                                 logger_prefix=consumer_params.logger_prefix,
                                                 create_logger_file=consumer_params.create_logger_file,
                                                 log_filename=consumer_params.log_filename,
+                                                logger_name=consumer_params.logger_name,
                                                 broker_exclusive_config=self.consumer_params.broker_exclusive_config)
         if is_main_process:
             self.logger.info(f'{self.queue_name} consumer 的消费者配置:\n {self.consumer_params.json_str_value()}')
@@ -229,7 +231,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         if logger_prefix != '':
             logger_prefix += '--'
             # logger_name = f'{logger_prefix}{self.__class__.__name__}--{concurrent_name}--{queue_name}--{self.consuming_function.__name__}'
-        logger_name = f'funboost.{logger_prefix}{self.__class__.__name__}--{self.queue_name}'
+        logger_name = self.consumer_params.logger_name or f'funboost.{logger_prefix}{self.__class__.__name__}--{self.queue_name}'
         self.logger_name = logger_name
         log_filename = self.consumer_params.log_filename or f'funboost.{self.queue_name}.log'
         self.logger = LogManager(logger_name, logger_cls=TaskIdLogger).get_logger_and_add_handlers(
@@ -317,7 +319,12 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
     def start_consuming_message(self):
         # ConsumersManager.show_all_consumer_info()
         # noinspection PyBroadException
-
+        pid_queue_name_tuple = (os.getpid(), self.queue_name)
+        if pid_queue_name_tuple in lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set:
+            self.logger.warning(f'{pid_queue_name_tuple} 已启动消费,不要一直去启动消费,funboost框架自动阻止.')  # 有的人乱写代码,无数次在函数内部或for循环里面执行 f.consume(),一个队列只需要启动一次消费,不然每启动一次性能消耗很大,直到程序崩溃
+            return
+        else:
+            lazy_impoter.BoostersManager.pid_queue_name__has_start_consume_set.add(pid_queue_name_tuple)
         GlobalVars.has_start_a_consumer_flag = True
         try:
             self._concurrent_mode_dispatcher.check_all_concurrent_mode()
@@ -364,7 +371,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         key = 'apscheduler.redisjobstore_runonce'
         if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):  # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
             cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
-            LazyImpoter().BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
+            lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
 
     @abc.abstractmethod
     def _shedual_task(self):

{funboost-43.0 → funboost-43.2}/funboost/core/booster.py

@@ -13,6 +13,7 @@ from funboost.core.exceptions import BoostDecoParamsIsOldVersion
 from funboost.core.func_params_model import BoosterParams, FunctionResultStatusPersistanceConfig, PriorityConsumingControlConfig
 
 from funboost.factories.consumer_factory import get_consumer
+from collections import defaultdict
 
 if typing.TYPE_CHECKING:
     from funboost.core.msg_result_getter import AsyncResult
@@ -180,6 +181,8 @@ class BoostersManager:
     # queue_name__boost_params_consuming_function_map 字典存放  {queue_name,(@boost的入参字典,@boost装饰的消费函数)}
     queue_name__boost_params_map = {}  # type: typing.Dict[str,BoosterParams]
 
+    pid_queue_name__has_start_consume_set = set()
+
     @classmethod
     def regist_booster(cls, queue_name: str, booster: Booster):
         """这个是框架在@boost时候自动调用的,无需用户亲自调用"""

{funboost-43.0 → funboost-43.2}/funboost/core/cli/discovery_boosters.py

@@ -7,7 +7,7 @@ import importlib.util
 # import nb_log
 from funboost.core.loggers import FunboostFileLoggerMixin
 from funboost.utils.decorators import flyweight
-from funboost.core.lazy_impoter import LazyImpoter
+from funboost.core.lazy_impoter import lazy_impoter
 
 @flyweight
 class BoosterDiscovery(FunboostFileLoggerMixin):
@@ -63,10 +63,12 @@ class BoosterDiscovery(FunboostFileLoggerMixin):
                 if Path(file_path) == Path(sys._getframe(1).f_code.co_filename):
                     self.logger.warning(f'排除导入调用auto_discovery的模块自身 {file_path}')  # 否则下面的import这个文件,会造成无限懵逼死循环
                     continue
-                spec = importlib.util.spec_from_file_location('module', file_path)
+
+                module_name = Path(file_path).as_posix().replace('/', '.') + '.' + Path(file_path).stem
+                spec = importlib.util.spec_from_file_location(module_name, file_path)
                 module = importlib.util.module_from_spec(spec)
                 spec.loader.exec_module(module)
-        LazyImpoter().BoostersManager.show_all_boosters()
+        lazy_impoter.BoostersManager.show_all_boosters()
 
 
 if __name__ == '__main__':

{funboost-43.0 → funboost-43.2}/funboost/core/current_task.py

@@ -47,7 +47,7 @@ if __name__ == '__main__':
     """
 
 
-class ThreadCurrentTask:
+class __ThreadCurrentTask:
     """
     用于在用户自己函数内部去获取 消息的完整体,当前重试次数等.
     """
@@ -92,7 +92,7 @@ class ThreadCurrentTask:
     def logger(self, logger: logging.Logger):
         self._fct_local_data.logger = logger
 
-
+thread_current_task = __ThreadCurrentTask()
 def is_asyncio_environment():
     try:
         loop = asyncio.get_running_loop()
@@ -101,7 +101,7 @@ def is_asyncio_environment():
         return False
 
 
-class AsyncioCurrentTask:
+class __AsyncioCurrentTask:
     _function_params = contextvars.ContextVar("function_params")
     _full_msg = contextvars.ContextVar("full_msg")
     _function_result_status = contextvars.ContextVar("function_result_status")
@@ -143,9 +143,10 @@ class AsyncioCurrentTask:
     def logger(self, logger: logging.Logger):
         self._logger.set(logger)
 
+asyncio_current_task = __AsyncioCurrentTask()
 
 def funboost_current_task():
-    return AsyncioCurrentTask() if is_asyncio_environment() else ThreadCurrentTask()
+    return asyncio_current_task if is_asyncio_environment() else thread_current_task
 
 
 def get_current_taskid():
@@ -153,7 +154,7 @@ def get_current_taskid():
     # return fct.function_result_status.task_id
     try:
         return fct.task_id  # 不在funboost的消费函数里面就获取不到上下文了
-    except AttributeError as e:
+    except (AttributeError,LookupError) as e:
         # print(e,type(e))
         return 'no_task_id'
 
@@ -162,5 +163,5 @@ if __name__ == '__main__':
     print()
     for i in range(100000):
         funboost_current_task()
-        print(get_current_taskid())
+        get_current_taskid()
     print()

{funboost-43.0 → funboost-43.2}/funboost/core/func_params_model.py

@@ -143,6 +143,7 @@ class BoosterParams(BaseJsonAbleModel):
     log_level: int = logging.DEBUG  # 消费者和发布者的日志级别,建议设置DEBUG级别,不然无法知道正在运行什么消息
     logger_prefix: str = ''  # 日志名字前缀,可以设置前缀
     create_logger_file: bool = True  # 发布者和消费者是否创建文件文件日志,为False则只打印控制台不写文件.
+    logger_name: str = ''  # 队列消费者发布者的日志命名空间.
     log_filename: typing.Union[str, None] = None  # 消费者发布者的文件日志名字.如果为None,则自动使用 funboost.队列 名字作为文件日志名字.  日志文件夹是在nb_log_config.py的 LOG_PATH中决定的.
     is_show_message_get_from_broker: bool = False  # 运行时候,是否记录从消息队列获取出来的消息内容
     is_print_detail_exception: bool = True  # 消费函数出错时候,是否打印详细的报错堆栈,为False则只打印简略的报错信息不包含堆栈.
@@ -248,6 +249,7 @@ class PublisherParams(BaseJsonAbleModel):
     log_level: int = logging.DEBUG
     logger_prefix: str = ''
     create_logger_file: bool = True
+    logger_name: str = ''  # 队列消费者发布者的日志命名空间.
     log_filename: typing.Optional[str] = None
     clear_queue_within_init: bool = False  # with 语法发布时候,先清空消息队列
     consuming_function: typing.Callable = None  # consuming_function 作用是 inspect 模块获取函数的入参信息

{funboost-43.0 → funboost-43.2}/funboost/core/lazy_impoter.py

@@ -18,8 +18,9 @@ class LazyImpoter:
     #     from funboost.core.current_task import get_current_taskid
     #     return get_current_taskid
 
+lazy_impoter = LazyImpoter()
 
 if __name__ == '__main__':
     for i in range(10000):
-        LazyImpoter().BoostersManager
-        LazyImpoter().BoostersManager
+        lazy_impoter.BoostersManager
+        lazy_impoter.BoostersManager

{funboost-43.0 → funboost-43.2}/funboost/core/muliti_process_enhance.py

@@ -7,11 +7,11 @@ from concurrent.futures import ProcessPoolExecutor
 from funboost.core.booster import Booster
 from funboost.core.helper_funs import run_forever
 from funboost.core.loggers import flogger
-from funboost.core.lazy_impoter import LazyImpoter
+from funboost.core.lazy_impoter import lazy_impoter
 
 
 def _run_consumer_in_new_process(queue_name, ):
-    booster_current_pid = LazyImpoter().BoostersManager.get_or_create_booster_by_queue_name(queue_name)
+    booster_current_pid = lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name)
     # booster_current_pid = boost(**boost_params)(consuming_function)
     booster_current_pid.consume()
     # ConsumersManager.join_all_consumer_shedual_task_thread()
@@ -49,7 +49,7 @@ def run_consumer_with_multi_process(booster: Booster, process_num=1):
 
 
 def _multi_process_pub_params_list_in_new_process(queue_name, msgs: List[dict]):
-    booster_current_pid = LazyImpoter().BoostersManager.get_or_create_booster_by_queue_name(queue_name)
+    booster_current_pid = lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name)
     publisher = booster_current_pid.publisher
     publisher.set_log_level(20)  # 超高速发布，如果打印详细debug日志会卡死屏幕和严重降低代码速度。
     for msg in msgs:

funboost-43.2/funboost/core/task_id_logger.py

@@ -0,0 +1,19 @@
+import functools
+
+from nb_log import CompatibleLogger
+from funboost.core.current_task import get_current_taskid
+
+
+class TaskIdLogger(CompatibleLogger):
+    """
+    如果你要使用带taskid的日志模板,一定要使用
+     LogManager('namexx',logger_cls=TaskIdLogger).get_logger_and_add_handlers(....)
+     的方式来创建logger, 就是需要指定logger_cls=TaskIdLogger ,否则的话你需要在打印日志时候 手动传递extra logger.info(msg,extra={'task_id':task_idxxx})
+     """
+    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
+        extra = extra or {}
+        if 'task_id' not in extra:
+            extra['task_id'] = get_current_taskid()
+        if 'sys_getframe_n' not in extra:
+            extra['sys_getframe_n'] = 3
+        super()._log(level, msg, args, exc_info, extra, stack_info)

{funboost-43.0 → funboost-43.2}/funboost/funboost_config_deafult.py

@@ -97,10 +97,10 @@ class BrokerConnConfig(DataClassBase):
 
 class FunboostCommonConfig(DataClassBase):
     # nb_log包的第几个日志模板，内置了7个模板，可以在你当前项目根目录下的nb_log_config.py文件扩展模板。
-    # NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER = 11  # 7是简短的不可跳转，5是可点击跳转的，11是可显示ip 进程 线程的模板。
+    # NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER = 11  # 7是简短的不可跳转，5是可点击跳转的，11是可显示ip 进程 线程的模板,也可以亲自设置日志模板不传递数字。
     NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER = logging.Formatter(
         f'%(asctime)s-({nb_log_config_default.computer_ip},{nb_log_config_default.computer_name})-[p%(process)d_t%(thread)d] - %(name)s - "%(filename)s:%(lineno)d" - %(funcName)s - %(levelname)s - %(task_id)s - %(message)s',
-        "%Y-%m-%d %H:%M:%S",)   # 这个是带task_id的日子模板,日子可以显示task_id,方便用户窗帘排查某一个任务的所有日志.
+        "%Y-%m-%d %H:%M:%S",)   # 这个是带task_id的日志模板,日志可以显示task_id,方便用户串联起来排查某一个人物消息的所有日志.
 
     TIMEZONE = 'Asia/Shanghai'  # 时区
 

{funboost-43.0 → funboost-43.2}/funboost/publishers/base_publisher.py

@@ -159,13 +159,13 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         logger_prefix = self.publisher_params.logger_prefix
         if logger_prefix != '':
             logger_prefix += '--'
-        logger_name = f'funboost.{logger_prefix}{self.__class__.__name__}--{self.queue_name}'
-        log_filename = self.publisher_params.log_filename or f'funboost.{self.queue_name}.log'
-        self.logger = nb_log.LogManager(logger_name,logger_cls=TaskIdLogger).get_logger_and_add_handlers(
-                                 log_level_int=self.publisher_params.log_level,
-                                 log_filename=log_filename if self.publisher_params.create_logger_file else None,
-                                 error_log_filename=nb_log.generate_error_file_name(log_filename),
-                                 formatter_template=FunboostCommonConfig.NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER, )
+        self.logger_name = self.publisher_params.logger_name or f'funboost.{logger_prefix}{self.__class__.__name__}--{self.queue_name}'
+        self.log_filename = self.publisher_params.log_filename or f'funboost.{self.queue_name}.log'
+        self.logger = nb_log.LogManager(self.logger_name, logger_cls=TaskIdLogger).get_logger_and_add_handlers(
+            log_level_int=self.publisher_params.log_level,
+            log_filename=self.log_filename if self.publisher_params.create_logger_file else None,
+            error_log_filename=nb_log.generate_error_file_name(self.log_filename),
+            formatter_template=FunboostCommonConfig.NB_LOG_FORMATER_INDEX_FOR_CONSUMER_AND_PUBLISHER, )
 
     def _init_count(self):
         self._current_time = time.time()
@@ -214,7 +214,7 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         decorators.handle_exception(retry_times=10, is_throw_error=True, time_sleep=0.1)(
             self.concrete_realization_of_publish)(json.dumps(msg, ensure_ascii=False))
 
-        self.logger.debug(f'向{self._queue_name} 队列，推送消息 耗时{round(time.time() - t_start, 4)}秒  {msg_function_kw}',extra={'task_id':task_id})  # 显示msg太长了。
+        self.logger.debug(f'向{self._queue_name} 队列，推送消息 耗时{round(time.time() - t_start, 4)}秒  {msg_function_kw}', extra={'task_id': task_id})  # 显示msg太长了。
         with self._lock_for_count:
             self.count_per_minute += 1
             self.publish_msg_num_total += 1
@@ -224,9 +224,9 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
                 self._init_count()
         return AsyncResult(task_id)
 
-    def send_msg(self,msg:typing.Union[dict,str]):
+    def send_msg(self, msg: typing.Union[dict, str]):
         """直接发送任意消息内容到消息队列,不生成辅助参数,无视函数入参名字,不校验入参个数和键名"""
-        if isinstance(msg,dict):
+        if isinstance(msg, dict):
             msg = json.dumps(msg, ensure_ascii=False)
         decorators.handle_exception(retry_times=10, is_throw_error=True, time_sleep=0.1)(
             self.concrete_realization_of_publish)(msg)

{funboost-43.0 → funboost-43.2}/funboost/utils/dependency_packages_in_pythonpath/func_timeout/dafunc.py

@@ -64,10 +64,10 @@ def func_timeout(timeout, func, args=(), kwargs=None):
     exception = []
     isStopped = False
 
-    from funboost.core.current_task import ThreadCurrentTask
+    from funboost.core.current_task import thread_current_task
 
     def funcwrap(args2, kwargs2,_fct_local_data_dict):
-        fct = ThreadCurrentTask()
+        fct = thread_current_task
         fct._fct_local_data.__dict__.update(_fct_local_data_dict)  # 把funboost的消费线程上下文需要传递到超时线程上下文里面来.
         try:
             ret.append( func(*args2, **kwargs2) )
@@ -86,7 +86,7 @@ def func_timeout(timeout, func, args=(), kwargs=None):
 
 
     # fct = funboost_current_task()
-    thread = StoppableThread(target=funcwrap, args=(args, kwargs,ThreadCurrentTask()._fct_local_data.__dict__))
+    thread = StoppableThread(target=funcwrap, args=(args, kwargs,thread_current_task._fct_local_data.__dict__))
     thread.daemon = True
 
     thread.start()

{funboost-43.0 → funboost-43.2/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 43.0
+Version: 43.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

funboost-43.0/funboost/core/task_id_logger.py

@@ -1,13 +0,0 @@
-import functools
-
-from nb_log import CompatibleLogger
-from funboost.core.current_task import get_current_taskid
-
-
-class TaskIdLogger(CompatibleLogger):
-    def _log(self, level, msg, args, exc_info=None, extra=None, stack_info=False):
-        extra = extra or {}
-        if 'task_id' not in extra:
-            extra['task_id'] = get_current_taskid()
-        extra['sys_getframe_n'] = 3
-        super()._log(level, msg, args, exc_info, extra, stack_info)