funboost 46.8__tar.gz → 46.9__tar.gz

{funboost-46.8/funboost.egg-info → funboost-46.9}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 46.8
+Version: 46.9
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-46.8 → funboost-46.9}/funboost/__init__.py

@@ -13,7 +13,7 @@ set_frame_config这个模块的 use_config_form_funboost_config_module() 是核
 这段注释说明和使用的用户无关,只和框架开发人员有关.
 '''
 
-__version__ = "46.8"
+__version__ = "46.9"
 
 from funboost.set_frame_config import show_frame_config
 

{funboost-46.8 → funboost-46.9}/funboost/concurrent_pool/custom_threadpool_executor.py

@@ -19,6 +19,8 @@
 
 可以在各种地方加入 time.sleep 来验证 第1条和第2条的自动智能缩放功能。
 """
+import logging
+
 import os
 import atexit
 import queue
@@ -175,6 +177,9 @@ class _CustomThread(threading.Thread, FunboostFileLoggerMixin, LoggerLevelSetter
     # noinspection PyProtectedMember
     def run(self):
         # noinspection PyUnresolvedReferences
+        # print(logging.getLogger(None).level,logging.getLogger(None).handlers)
+        # print(self.logger.level)
+        # print(self.logger.handlers)
         self.logger.debug(f'新启动线程 {self._ident} ')
         self._executorx._change_threads_free_count(1)
         while True:

{funboost-46.8 → funboost-46.9}/funboost/consumers/base_consumer.py

@@ -13,6 +13,7 @@ import typing
 import abc
 import copy
 from apscheduler.jobstores.memory import MemoryJobStore
+from funboost.core.funboost_time import FunboostTime
 from pathlib import Path
 # from multiprocessing import Process
 import datetime
@@ -401,11 +402,12 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
     logger_apscheduler = get_logger('push_for_apscheduler_use_database_store', log_filename='push_for_apscheduler_use_database_store.log')
 
     @classmethod
-    def _push_for_apscheduler_use_database_store(cls, queue_name, msg, runonce_uuid):
-        key = 'apscheduler.redisjobstore_runonce'
-        if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):  # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
-            cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
-            funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
+    def _push_apscheduler_task_to_broker(cls, queue_name, msg, runonce_uuid):
+        funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
+        # key = 'apscheduler.redisjobstore_runonce'
+        # if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):  # 这样可以阻止多次启动同队列名消费者 redis jobstore多次运行函数.
+        #     cls.logger_apscheduler.debug(f'延时任务用普通消息重新发布到普通队列 {msg}')
+        #     funboost_lazy_impoter.BoostersManager.get_or_create_booster_by_queue_name(queue_name).publish(msg)
 
     @abc.abstractmethod
     def _shedual_task(self):
@@ -486,9 +488,10 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         run_date = None
         # print(kw)
         if msg_countdown:
-            run_date = time_util.DatetimeConverter(kw['body']['extra']['publish_time']).datetime_obj + datetime.timedelta(seconds=msg_countdown)
+            run_date = FunboostTime(kw['body']['extra']['publish_time']).datetime_obj + datetime.timedelta(seconds=msg_countdown)
         if msg_eta:
-            run_date = time_util.DatetimeConverter(msg_eta).datetime_obj
+
+            run_date = FunboostTime(msg_eta).datetime_obj
         # print(run_date,time_util.DatetimeConverter().datetime_obj)
         # print(run_date.timestamp(),time_util.DatetimeConverter().datetime_obj.timestamp())
         # print(self.concurrent_pool)
@@ -507,7 +510,7 @@ class AbstractConsumer(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             self.__delete_eta_countdown(msg_no_delay)
             # print(msg_no_delay)
             # 数据库作为apscheduler的jobstores时候， 不能用 self.pbulisher_of_same_queue.publish，self不能序列化
-            self._delay_task_scheduler.add_job(self._push_for_apscheduler_use_database_store, 'date', run_date=run_date,
+            self._delay_task_scheduler.add_job(self._push_apscheduler_task_to_broker, 'date', run_date=run_date,
                                                kwargs={'queue_name': self.queue_name, 'msg': msg_no_delay, 'runonce_uuid': str(uuid.uuid4())},
                                                misfire_grace_time=misfire_grace_time,
                                               )

{funboost-46.8 → funboost-46.9}/funboost/core/func_params_model.py

@@ -194,7 +194,7 @@ class BoosterParams(BaseJsonAbleModel):
     broker_exclusive_config: dict = {}  # 加上一个不同种类中间件非通用的配置,不同中间件自身独有的配置，不是所有中间件都兼容的配置，因为框架支持30种消息队列，消息队列不仅仅是一般的先进先出queue这么简单的概念，
     # 例如kafka支持消费者组，rabbitmq也支持各种独特概念例如各种ack机制 复杂路由机制，有的中间件原生能支持消息优先级有的中间件不支持,每一种消息队列都有独特的配置参数意义，可以通过这里传递。每种中间件能传递的键值对可以看consumer类的 BROKER_EXCLUSIVE_CONFIG_DEFAULT
 
-    should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字,  如果消费函数你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
+    should_check_publish_func_params: bool = True  # 消息发布时候是否校验消息发布内容,比如有的人发布消息,函数只接受a,b两个入参,他去传2个入参,或者传参不存在的参数名字; 如果消费函数加了装饰器 ，你非要写*args,**kwargs,那就需要关掉发布消息时候的函数入参检查
 
     consumer_override_cls: typing.Optional[typing.Type] = None  # 使用 consumer_override_cls 和 publisher_override_cls 来自定义重写或新增消费者 发布者,见文档4.21b介绍，
     publisher_override_cls: typing.Optional[typing.Type] = None
@@ -283,7 +283,7 @@ class PriorityConsumingControlConfig(BaseJsonAbleModel):
     is_using_rpc_mode: bool = None
 
     countdown: typing.Union[float, int] = None
-    eta: datetime.datetime = None
+    eta: typing.Union[datetime.datetime, str] = None  # 时间对象， 或 %Y-%m-%d %H:%M:%S 字符串。
     misfire_grace_time: typing.Union[int, None] = None
 
     other_extra_params: dict = None  # 其他参数, 例如消息优先级 , priority_control_config=PriorityConsumingControlConfig(other_extra_params={'priroty': priorityxx})，

{funboost-46.8 → funboost-46.9}/funboost/timing_job/__init__.py

@@ -2,35 +2,29 @@
 集成定时任务。
 """
 import atexit
-import copy
-import importlib
-
-import pickle
 
 import time
 from apscheduler.executors.pool import BasePoolExecutor
 
-from funboost.utils.decorators import RedisDistributedLockContextManager,RedisDistributedBlockLockContextManager
 from typing import Union
 import threading
 
 from apscheduler.schedulers.background import BackgroundScheduler
-from apscheduler.jobstores.redis import RedisJobStore
 # noinspection PyProtectedMember
 from apscheduler.schedulers.base import STATE_STOPPED, STATE_RUNNING
-from apscheduler.util import undefined,TIMEOUT_MAX
+from apscheduler.util import undefined, TIMEOUT_MAX
 import deprecated
 from funboost.utils.redis_manager import RedisMixin
 
-from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
+from funboost.funboost_config_deafult import FunboostCommonConfig
 
 from funboost.consumers.base_consumer import AbstractConsumer
 from funboost.core.booster import BoostersManager, Booster
-from funboost.publishers.base_publisher import AbstractPublisher
+
 from funboost import BoosterParams
-from funboost.concurrent_pool.flexible_thread_pool import FlexibleThreadPool
 from funboost.concurrent_pool.custom_threadpool_executor import ThreadPoolExecutorShrinkAble
 
+
 @deprecated.deprecated(reason='以后不要再使用这种方式，对于job_store为数据库时候需要序列化不好。使用内存和数据库都兼容的添加任务方式: add_push_job')
 def timing_publish_deco(consuming_func_decorated_or_consumer: Union[callable, AbstractConsumer]):
     def _deco(*args, **kwargs):
@@ -49,14 +43,16 @@ def push_fun_params_to_broker(queue_name: str, *args, runonce_uuid=None, **kwarg
     queue_name 队列名字
     *args **kwargs 是消费函数的入参
     发布消息中可以包括,runonce_uuid这个入参,确保分布式多个脚本都启动了定时器,导致每个定时器重复发布到消息队列,值你自己写  str(uuid.uuid4())
+    # 不需要传递 runonce_uuid 了，已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了。
     """
-    if runonce_uuid:
+    if runonce_uuid:   # 不需要传递 runonce_uuid 了，已经用专门的 FunboostBackgroundSchedulerProcessJobsWithinRedisLock 解决了 process_jobs的问题。
         key = 'apscheduler.redisjobstore_runonce2'
         if RedisMixin().redis_db_frame.sadd(key, runonce_uuid):
             BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
     else:
         BoostersManager.get_or_create_booster_by_queue_name(queue_name).push(*args, **kwargs)
 
+
 class ThreadPoolExecutorForAps(BasePoolExecutor):
     """
     An executor that runs jobs in a concurrent.futures thread pool.
@@ -68,10 +64,11 @@ class ThreadPoolExecutorForAps(BasePoolExecutor):
         ThreadPoolExecutor constructor
     """
 
-    def __init__(self, max_workers=10, ):
-        pool = ThreadPoolExecutorShrinkAble(int(max_workers),)
+    def __init__(self, max_workers=10, pool_kwargs=None):
+        pool = ThreadPoolExecutorShrinkAble(int(max_workers), )
         super().__init__(pool)
 
+
 class FunboostBackgroundScheduler(BackgroundScheduler):
     """
     自定义的， 继承了官方BackgroundScheduler，
@@ -126,7 +123,7 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
         args_list.insert(0, func.queue_name)
         args = tuple(args_list)
         kwargs = kwargs or {}
-        kwargs['runonce_uuid'] = runonce_uuid
+        kwargs['runonce_uuid'] = runonce_uuid  # 忽略，用户不需要传递runonce_uuid入参。
         return self.add_job(push_fun_params_to_broker, trigger, args, kwargs, id, name,
                             misfire_grace_time, coalesce, max_instances,
                             next_run_time, jobstore, executor,
@@ -173,7 +170,7 @@ class FunboostBackgroundScheduler(BackgroundScheduler):
             if wait_seconds is None:
                 wait_seconds = MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS
             self._last_wait_seconds = min(wait_seconds, MAX_WAIT_SECONDS_FOR_NEX_PROCESS_JOBS)
-            if wait_seconds in (None,TIMEOUT_MAX):
+            if wait_seconds in (None, TIMEOUT_MAX):
                 self._last_has_task = False
             else:
                 self._last_has_task = True

{funboost-46.8 → funboost-46.9}/funboost/utils/time_util.py

@@ -5,7 +5,7 @@ import datetime
 import time
 import re
 import pytz
-
+from funboost.core.funboost_time import FunboostTime
 
 from funboost.utils import nb_print
 
@@ -75,21 +75,22 @@ class DatetimeConverter:
         """
         :param datetimex: 接受时间戳  datatime类型 和 时间字符串三种类型
         """
-        if isinstance(datetimex, str):
-            if not re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', datetimex):
-                raise ValueError('时间字符串的格式不符合此传参的规定')
-            else:
-                self.datetime_obj = datetime.datetime.strptime(datetimex, self.DATETIME_FORMATTER)
-        elif isinstance(datetimex, (int, float)):
-            if datetimex < 1:
-                datetimex += 86400
-            self.datetime_obj = datetime.datetime.fromtimestamp(datetimex, tz=pytz.timezone(_get_funboost_timezone()))  # 时间戳0在windows会出错。
-        elif isinstance(datetimex, datetime.datetime):
-            self.datetime_obj = datetimex
-        elif datetimex is None:
-            self.datetime_obj = datetime.datetime.now(tz=pytz.timezone(_get_funboost_timezone()))
-        else:
-            raise ValueError('实例化时候的传参不符合规定')
+        # if isinstance(datetimex, str):
+        #     if not re.match(r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}', datetimex):
+        #         raise ValueError('时间字符串的格式不符合此传参的规定')
+        #     else:
+        #         self.datetime_obj = datetime.datetime.strptime(datetimex, self.DATETIME_FORMATTER)
+        # elif isinstance(datetimex, (int, float)):
+        #     if datetimex < 1:
+        #         datetimex += 86400
+        #     self.datetime_obj = datetime.datetime.fromtimestamp(datetimex, tz=pytz.timezone(_get_funboost_timezone()))  # 时间戳0在windows会出错。
+        # elif isinstance(datetimex, datetime.datetime):
+        #     self.datetime_obj = datetimex
+        # elif datetimex is None:
+        #     self.datetime_obj = datetime.datetime.now(tz=pytz.timezone(_get_funboost_timezone()))
+        # else:
+        #     raise ValueError('实例化时候的传参不符合规定')
+        self.datetime_obj = FunboostTime(datetimex).datetime_obj
 
     @property
     def datetime_str(self):

{funboost-46.8 → funboost-46.9/funboost.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: funboost
-Version: 46.8
+Version: 46.9
 Summary: pip install funboost，python全功能分布式函数调度框架,funboost的功能是全面性重量级，用户能想得到的功能99%全都有;funboost的使用方式是轻量级，只有@boost一行代码需要写。支持python所有类型的并发模式和一切知名消息队列中间件，支持如 celery dramatiq等框架整体作为funboost中间件，python函数加速器，框架包罗万象，用户能想到的控制功能全都有。一统编程思维，兼容50% python业务场景，适用范围广。只需要一行代码即可分布式执行python一切函数，99%用过funboost的pythoner 感受是　简易 方便 强劲 强大，相见恨晚 
 Home-page: https://github.com/ydf0509/funboost
 Author: bfzs

{funboost-46.8 → funboost-46.9}/funboost.egg-info/requires.txt

@@ -1,6 +1,6 @@
-nb_log>=12.6
+nb_log>=13.2
 nb_libs>=1.8
-nb_time>=1.8
+nb_time>=2.0
 pymongo==4.3.3
 AMQPStorm==2.10.6
 rabbitpy==2.0.1

{funboost-46.8 → funboost-46.9}/setup.py

@@ -77,9 +77,9 @@ setup(
         'Topic :: Software Development :: Libraries'
     ],
     install_requires=[
-        'nb_log>=12.6',
+        'nb_log>=13.2',
         'nb_libs>=1.8',
-        'nb_time>=1.8',
+        'nb_time>=2.0',
         'pymongo==4.3.3',  # 3.5.1  -> 4.0.2
         'AMQPStorm==2.10.6',
         'rabbitpy==2.0.1',