funboost 42.0__tar.gz → 42.2__tar.gz

{funboost-42.0/funboost.egg-info → funboost-42.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 42.0
+Version: 42.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -147,11 +147,15 @@ pip install funboost --upgrade
 [//]: # ()
 [//]: # ([img-46.png](https://postimg.cc/hfW0VhCX))
 
-funboosts示图：
+funboost示图：
+![](https://i.niupic.com/images/2024/02/03/g3tr.png)
+
 
 [//]: # ([![img-46.png](https://i.postimg.cc/tC7mQxWN/img-46.png)](https://postimg.cc/hfW0VhCX))
-就是最普通的生产者消费者流程图
-![](https://i.niupic.com/images/2023/12/18/dVaP.png)
+
+[//]: # (就是最普通的生产者消费者流程图)
+
+[//]: # (![](https://i.niupic.com/images/2023/12/18/dVaP.png))
 
 也就是这种非常普通的流程图,一样的意思
 

{funboost-42.0 → funboost-42.2}/README.md

@@ -116,11 +116,15 @@ pip install funboost --upgrade
 [//]: # ()
 [//]: # ([img-46.png](https://postimg.cc/hfW0VhCX))
 
-funboosts示图：
+funboost示图：
+![](https://i.niupic.com/images/2024/02/03/g3tr.png)
+
 
 [//]: # ([![img-46.png](https://i.postimg.cc/tC7mQxWN/img-46.png)](https://postimg.cc/hfW0VhCX))
-就是最普通的生产者消费者流程图
-![](https://i.niupic.com/images/2023/12/18/dVaP.png)
+
+[//]: # (就是最普通的生产者消费者流程图)
+
+[//]: # (![](https://i.niupic.com/images/2023/12/18/dVaP.png))
 
 也就是这种非常普通的流程图,一样的意思
 

{funboost-42.0 → funboost-42.2}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "42.0"
+__version__ = "42.2"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-42.0 → funboost-42.2}/funboost/assist/celery_helper.py

@@ -7,6 +7,7 @@ from functools import partial
 
 import celery
 
+import nb_log
 from funboost.funboost_config_deafult import BrokerConnConfig,FunboostCommonConfig
 from funboost import  ConcurrentModeEnum
 from funboost.core.loggers import get_funboost_file_logger,get_logger
@@ -109,4 +110,14 @@ class CeleryHelper:
         使用nb_log的日志来取代celery的日志
         """
         celery_app.conf.worker_hijack_root_logger = False
+        # logging.getLogger('celery').handlers=[]
+        # logging.getLogger('celery.worker.strategy').handlers = []
+        # logging.getLogger('celery.app.trace').handlers = []
+        # logging.getLogger('celery.worker').handlers = []
+        # logging.getLogger('celery.app').handlers = []
+        # logging.getLogger().handlers=[]
         get_logger('celery', log_level_int=log_level, log_filename=log_filename, formatter_template=formatter_template, )
+        get_logger(None, log_level_int=logging.WARNING, log_filename=log_filename, formatter_template=formatter_template, )
+        for name in ['celery','celery.worker.strategy','celery.app.trace','celery.worker','celery.app',None]:
+            nb_log.LogManager(name).prevent_add_handlers()
+        nb_log.LogManager(None).preset_log_level(logging.WARNING)

{funboost-42.0 → funboost-42.2}/funboost/constant.py

@@ -14,7 +14,7 @@ class BrokerEnum:
 
     MONGOMQ = 'MONGOMQ'  # 使用mongo的表中的行模拟的 作为分布式消息队列，支持消费确认。
 
-    SQLITE_QUEUE = 'sqlite3'  # 使用基于sqlute3模拟消息队列，支持消费确认和持久化，但不支持跨机器共享任务，可以基于本机单机跨脚本和跨进程共享任务，好处是不需要安装中间件。
+    SQLITE_QUEUE = 'sqlite3'  # 使用基于sqlite3模拟消息队列，支持消费确认和持久化，但不支持跨机器共享任务，可以基于本机单机跨脚本和跨进程共享任务，好处是不需要安装中间件。
     PERSISTQUEUE = SQLITE_QUEUE  # PERSISTQUEUE的别名
 
     NSQ = 'NSQ'  # 基于nsq作为分布式消息队列，支持消费确认。

{funboost-42.0 → funboost-42.2}/funboost/consumers/base_consumer.py

@@ -626,8 +626,8 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             if self.consumer_params.log_level <= logging.DEBUG:
                 result_str_to_be_print = str(function_result_status.result)[:100] if len(str(function_result_status.result)) < 100 else str(function_result_status.result)[:100] + '  。。。。。  '
                 self.logger.debug(f' 函数 {self.consuming_function.__name__}  '
-                                  f'第{current_retry_times + 1}次 运行, 正确了，函数运行时间是 {round(time.time() - t_start, 4)} 秒,入参是 {function_only_params}  '
-                                  f' 结果是  {result_str_to_be_print} ，  {self._get_concurrent_info()}  ')
+                                  f'第{current_retry_times + 1}次 运行, 正确了，函数运行时间是 {round(time.time() - t_start, 4)} 秒,入参是 {function_only_params} , '
+                                  f'结果是  {result_str_to_be_print}   {self._get_concurrent_info()}  ')
         except BaseException as e:
             if isinstance(e, (ExceptionForRequeue,)):  # mongo经常维护备份时候插入不了或挂了，或者自己主动抛出一个ExceptionForRequeue类型的错误会重新入队，不受指定重试次数逇约束。
                 log_msg = f'函数 [{self.consuming_function.__name__}] 中发生错误 {type(e)}  {e} 。消息重新放入当前队列 {self._queue_name}'

funboost-42.2/funboost/contrib/django_db_deco.py

@@ -0,0 +1,24 @@
+from django.db import close_old_connections
+
+
+def close_old_connections_deco(f):
+    """
+    如果是消费函数里面需要操作django orm,那么请写上 consumin_function_decorator=close_old_connections_deco
+    @boost(BoosterParams(queue_name='create_student_queue',
+                         broker_kind=BrokerEnum.REDIS_ACK_ABLE,
+                         consumin_function_decorator=close_old_connections_deco, # 如果gone away 一直好不了,可以加这个装饰器. django_celery django-apschrduler 这些源码中 也是调用了 close_old_connections_deco方法.
+
+                         )
+           )
+    """
+
+    def _inner(*args, **kwargs):
+        close_old_connections()
+        try:
+            result = f(*args, **kwargs)
+        finally:
+            close_old_connections()
+
+        return result
+
+    return _inner

funboost-42.2/funboost/contrib/save_result_status_to_sqldb.py

@@ -0,0 +1,114 @@
+import copy
+import functools
+import json
+
+from db_libs.sqla_lib import SqlaReflectHelper
+from sqlalchemy import create_engine
+
+from funboost import boost, FunctionResultStatus, funboost_config_deafult
+
+"""
+-- 如果用户是先保存到mysql中而非mongodb,用户自己先创建表,用于保存函数消费状态和结果.
+
+CREATE TABLE funboost_consume_results
+(
+    _id              VARCHAR(255),
+    `function`         VARCHAR(255),
+    host_name        VARCHAR(255),
+    host_process     VARCHAR(255),
+    insert_minutes   VARCHAR(255),
+    insert_time      datetime,
+    insert_time_str  VARCHAR(255),
+    msg_dict         JSON,
+    params           JSON,
+    params_str       VARCHAR(255),
+    process_id       BIGINT(20),
+    publish_time     FLOAT,
+    publish_time_str VARCHAR(255),
+    queue_name       VARCHAR(255),
+    result           VARCHAR(255),
+    run_times        INT,
+    script_name      VARCHAR(255),
+    script_name_long VARCHAR(255),
+    success          BOOLEAN,
+    task_id          VARCHAR(255),
+    thread_id        BIGINT(20),
+    time_cost        FLOAT,
+    time_end         FLOAT,
+    time_start       FLOAT,
+    total_thread     INT,
+    utime            VARCHAR(255),
+    `exception`       MEDIUMTEXT ,
+    rpc_result_expire_seconds BIGINT(20),
+    primary key (_id),
+    key idx_insert_time(insert_time),
+    key idx_queue_name_insert_time(queue_name,insert_time),
+    key idx_params_str(params_str)
+)
+
+
+
+"""
+
+
+def _gen_insert_sql_and_values_by_dict(dictx: dict):
+    key_list = [f'`{k}`' for k in dictx.keys()]
+    fields = ", ".join(key_list)
+
+    # 构建占位符字符串
+    placeholders = ", ".join(['%s'] * len(dictx))
+
+    # 构建插入语句
+    insert_sql = f"INSERT INTO funboost_consume_results ({fields}) VALUES ({placeholders})"
+
+    # 获取数据字典的值作为插入的值
+    values = tuple(dictx.values())
+    values_new = tuple([json.dumps(v) if isinstance(v, dict) else v for v in values])
+    return insert_sql, values_new
+
+
+def _gen_insert_sqlalchemy(dictx: dict):
+    key_list = [f'`{k}`' for k in dictx.keys()]
+    fields = ", ".join(key_list)
+
+    value_list = dictx.keys()
+    value_list_2 = [f':{f}' for f in value_list]
+    values = ", ".join(value_list_2)
+
+    # 构建插入语句
+    insert_sql = f"INSERT INTO funboost_consume_results ({fields}) VALUES ({values})"
+
+    return insert_sql
+
+
+@functools.lru_cache()
+def get_sqla_helper():
+    enginex = create_engine(
+        funboost_config_deafult.BrokerConnConfig.SQLACHEMY_ENGINE_URL,
+        max_overflow=10,  # 超过连接池大小外最多创建的连接
+        pool_size=50,  # 连接池大小
+        pool_timeout=30,  # 池中没有线程最多等待的时间，否则报错
+        pool_recycle=3600,  # 多久之后对线程池中的线程进行一次连接的回收（重置）
+        echo=True)
+    sqla_helper = SqlaReflectHelper(enginex)
+    t_funboost_consume_results = sqla_helper.base_classes.funboost_consume_results
+    return enginex, sqla_helper, t_funboost_consume_results
+
+
+def save_result_status_to_sqlalchemy(function_result_status: FunctionResultStatus):
+    """ function_result_status变量上有各种丰富的信息 ,用户可以使用其中的信息
+    用户自定义记录函数消费信息的钩子函数
+
+    例如  @boost('test_user_custom', user_custom_record_process_info_func=save_result_status_to_sqlalchemy)
+    """
+    enginex, sqla_helper, t_funboost_consume_results = get_sqla_helper()
+
+    with sqla_helper.session as ss:
+        status_dict = function_result_status.get_status_dict()
+        status_dict_new = copy.copy(status_dict)
+        for k, v in status_dict.items():
+            if isinstance(v, dict):
+                status_dict_new[k] = json.dumps(v)
+        # sql = _gen_insert_sqlalchemy(status_dict) # 这种是sqlahemy sql方式插入.
+        # ss.execute(sql, status_dict_new)
+        ss.merge(t_funboost_consume_results(**status_dict_new)) # 这种是orm方式插入.

{funboost-42.0 → funboost-42.2}/funboost/core/cli/funboost_fire.py

@@ -85,6 +85,24 @@ class BoosterFire(object):
             BoostersManager.get_booster(queue_anme).consume()
         ctrl_c_recv()
 
+    def consume_all_queues(self,):
+        """
+        启动所有消息队列名的消费,无需指定队列名;
+        例子: consume_all_queues
+        """
+        for queue_anme in BoostersManager.get_all_queues():
+            BoostersManager.get_booster(queue_anme).consume()
+        ctrl_c_recv()
+
+    def multi_process_consume_all_queues(self,process_num):
+        """
+        启动所有消息队列名的消费,无需指定队列名,每个队列启动n个单独的消费进程;
+        例子: multi_process_consume_all_queues 2
+        """
+        for queue_anme in BoostersManager.get_all_queues():
+            BoostersManager.get_booster(queue_anme).multi_process_consume(process_num)
+        ctrl_c_recv()
+
     def multi_process_consume(self, **queue_name__process_num):
         """
         使用多进程启动消费,每个队列开启多个单独的进程消费;

{funboost-42.0 → funboost-42.2}/funboost/core/exceptions.py

@@ -1,16 +1,22 @@
-class ExceptionForRetry(Exception):
+
+
+class FunboostException(Exception):
+    """funboost 异常基类"""
+
+
+class ExceptionForRetry(FunboostException):
     """为了重试的，抛出错误。只是定义了一个子类，用不用都可以，函数出任何类型错误了框架都会自动重试"""
 
 
-class ExceptionForRequeue(Exception):
+class ExceptionForRequeue(FunboostException):
     """框架检测到此错误，重新放回当前队列中"""
 
 
-class ExceptionForPushToDlxqueue(Exception):
+class ExceptionForPushToDlxqueue(FunboostException):
     """框架检测到ExceptionForPushToDlxqueue错误，发布到死信队列"""
 
 
-class BoostDecoParamsIsOldVersion(Exception):
+class BoostDecoParamsIsOldVersion(FunboostException):
     new_version_change_hint = """
 你的@boost入参是老的方式,建议用新的入参方式,老入参方式不再支持函数入参代码自动补全了。
 

{funboost-42.0 → funboost-42.2}/funboost/core/loggers.py

@@ -42,7 +42,7 @@ class FunboostMetaTypeFileLogger(type):
 
 flogger = get_funboost_file_logger('funboost', )
 
-print(_try_get_user_funboost_common_config('FUNBOOST_PROMPT_LOG_LEVEL'))
+# print(_try_get_user_funboost_common_config('FUNBOOST_PROMPT_LOG_LEVEL'))
 logger_prompt = get_funboost_file_logger('funboost.prompt', log_level_int=_try_get_user_funboost_common_config('FUNBOOST_PROMPT_LOG_LEVEL') or logging.DEBUG)  # funboost框架的提示,用户自己可以设置日志级别
 nb_log.LogManager('_KeepAliveTimeThread').preset_log_level(_try_get_user_funboost_common_config('KEEPALIVETIMETHREAD_LOG_LEVEL') or logging.DEBUG)
 

{funboost-42.0 → funboost-42.2/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 42.0
+Version: 42.2
 Summary: pip install funboost，python全功能分布式函数调度框架,。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs
@@ -147,11 +147,15 @@ pip install funboost --upgrade
 [//]: # ()
 [//]: # ([img-46.png](https://postimg.cc/hfW0VhCX))
 
-funboosts示图：
+funboost示图：
+![](https://i.niupic.com/images/2024/02/03/g3tr.png)
+
 
 [//]: # ([![img-46.png](https://i.postimg.cc/tC7mQxWN/img-46.png)](https://postimg.cc/hfW0VhCX))
-就是最普通的生产者消费者流程图
-![](https://i.niupic.com/images/2023/12/18/dVaP.png)
+
+[//]: # (就是最普通的生产者消费者流程图)
+
+[//]: # (![](https://i.niupic.com/images/2023/12/18/dVaP.png))
 
 也就是这种非常普通的流程图,一样的意思
 

{funboost-42.0 → funboost-42.2}/funboost.egg-info/SOURCES.txt

@@ -85,8 +85,10 @@ funboost/consumers/udp_consumer.py
 funboost/consumers/zeromq_consumer.py
 funboost/contrib/__init__.py
 funboost/contrib/api_publish_msg.py
+funboost/contrib/django_db_deco.py
 funboost/contrib/queue2queue.py
 funboost/contrib/redis_consume_latest_msg_broker.py
+funboost/contrib/save_result_status_to_sqldb.py
 funboost/core/__init__.py
 funboost/core/active_cousumer_info_getter.py
 funboost/core/booster.py

{funboost-42.0 → funboost-42.2}/funboost.egg-info/requires.txt

@@ -47,7 +47,7 @@ confluent_kafka==1.7.0
 celery
 flower
 nameko==2.14.1
-sqlalchemy==1.4.8
+sqlalchemy==1.4.13
 sqlalchemy_utils==0.36.1
 dramatiq==1.14.2
 huey==2.4.5
@@ -62,7 +62,7 @@ confluent_kafka==1.7.0
 celery
 flower
 nameko==2.14.1
-sqlalchemy==1.4.8
+sqlalchemy==1.4.13
 sqlalchemy_utils==0.36.1
 dramatiq==1.14.2
 huey==2.4.5

{funboost-42.0 → funboost-42.2}/setup.py

@@ -7,7 +7,7 @@ extra_brokers = ['confluent_kafka==1.7.0',
                  'celery',
                  'flower',
                  'nameko==2.14.1',
-                 'sqlalchemy==1.4.8',
+                 'sqlalchemy==1.4.13',
                  'sqlalchemy_utils==0.36.1',
                  'dramatiq==1.14.2',
                  'huey==2.4.5',