funboost 49.5__py3-none-any.whl → 49.7__py3-none-any.whl

funboost/core/msg_result_getter.py

@@ -12,13 +12,29 @@ from funboost.utils.redis_manager import RedisMixin
 from funboost.utils.redis_manager import AioRedisMixin
 from funboost.core.serialization import Serialization
 
+from funboost.core.function_result_status_saver import FunctionResultStatus
+
 class HasNotAsyncResult(Exception):
     pass
 
 
 NO_RESULT = 'no_result'
 
-
+def _judge_rpc_function_result_status_obj(status_and_result_obj:FunctionResultStatus,raise_exception:bool):
+    if status_and_result_obj is None:
+        raise FunboostWaitRpcResultTimeout(f'等待 {status_and_result_obj.task_id} rpc结果超过了指定时间')
+    if status_and_result_obj.success is True:
+        return status_and_result_obj
+    else:
+        raw_erorr = status_and_result_obj.exception
+        if status_and_result_obj.exception_type == 'FunboostRpcResultError':
+            raw_erorr = json.loads(status_and_result_obj.exception_msg) # 使canvas链式报错json显示更美观
+        error_msg_dict = {'task_id':status_and_result_obj.task_id,'raw_error':raw_erorr}
+        if raise_exception:
+            raise FunboostRpcResultError(json.dumps(error_msg_dict,indent=4,ensure_ascii=False))
+        else:
+            status_and_result_obj.rpc_chain_error_msg_dict = error_msg_dict
+            return status_and_result_obj
 class AsyncResult(RedisMixin):
     default_callback_run_executor = FlexibleThreadPoolMinWorkers0(200,work_queue_maxsize=50)
 
@@ -63,6 +79,14 @@ class AsyncResult(RedisMixin):
                 return self._status_and_result
             return None
         return self._status_and_result
+    
+    @property
+    def status_and_result_obj(self) -> FunctionResultStatus:
+        """这个是为了比字典有更好的ide代码补全效果"""
+        if self.status_and_result is not None:
+            return FunctionResultStatus.parse_status_and_result_to_obj(self.status_and_result)
+    
+    rpc_data =status_and_result_obj
 
     def get(self):
         # print(self.status_and_result)
@@ -104,6 +128,14 @@ class AsyncResult(RedisMixin):
             async_result.set_callback(show_result) # 使用回调函数在线程池中并发的运行函数结果
         '''
         self.callback_run_executor.submit(self._run_callback_func, callback_func)
+    
+    def wait_rpc_data_or_raise(self,raise_exception:bool=True)->FunctionResultStatus:
+        return _judge_rpc_function_result_status_obj(self.status_and_result_obj,raise_exception)
+    
+    @classmethod
+    def batch_wait_rpc_data_or_raise(cls,r_list:typing.List['AsyncResult'],raise_exception:bool=True)->typing.List[FunctionResultStatus]:
+        return [ _judge_rpc_function_result_status_obj(r.status_and_result_obj,raise_exception) 
+                for r in r_list]
 
 
 class AioAsyncResult(AioRedisMixin):
@@ -169,6 +201,14 @@ if __name__ == '__main__':
             return None
         return self._status_and_result
 
+    @property
+    async def status_and_result_obj(self) -> FunctionResultStatus:
+        """这个是为了比字典有更好的ide代码补全效果"""
+        sr = await self.status_and_result
+        if sr is not None:
+            return FunctionResultStatus.parse_status_and_result_to_obj(sr)
+
+    rpc_data =status_and_result_obj
     async def get(self):
         # print(self.status_and_result)
         if (await self.status_and_result) is not None:
@@ -192,6 +232,16 @@ if __name__ == '__main__':
     async def set_callback(self, aio_callback_func: typing.Callable):
         asyncio.create_task(self._run_callback_func(callback_func=aio_callback_func))
 
+    async def wait_rpc_data_or_raise(self,raise_exception:bool=True)->FunctionResultStatus:
+        return _judge_rpc_function_result_status_obj(await self.status_and_result_obj,raise_exception)
+
+    @classmethod
+    async def batch_wait_rpc_data_or_raise(cls,r_list:typing.List['AioAsyncResult'],raise_exception:bool=True)->typing.List[FunctionResultStatus]:
+        return [ _judge_rpc_function_result_status_obj(await r.status_and_result_obj,raise_exception) 
+                for r in r_list]
+    
+
+
 
 class ResultFromMongo(MongoMixin):
     """

funboost/core/serialization.py

@@ -1,6 +1,9 @@
 import typing
-
+import json
 import orjson
+import pickle
+import ast
+
 class Serialization:
     @staticmethod
     def to_json_str(dic:typing.Union[dict,str]):
@@ -14,3 +17,27 @@ class Serialization:
         if isinstance(strx,dict):
             return strx
         return orjson.loads(strx)
+    
+    @staticmethod
+    def find_can_not_json_serializable_keys(dic:dict)->typing.List[str]:
+        can_not_json_serializable_keys = []
+        dic = Serialization.to_dict(dic)
+        for k,v in dic.items():
+            if not isinstance(v,str):
+                try:
+                    json.dumps(v)
+                except:
+                    can_not_json_serializable_keys.append(k)
+        return can_not_json_serializable_keys
+    
+
+class PickleHelper:
+    @staticmethod
+    def to_str(obj_x:typing.Any):
+        return str(pickle.dumps(obj_x)) # 对象pickle,转成字符串
+    
+    @staticmethod
+    def to_obj(str_x:str):
+        return pickle.loads(ast.literal_eval(str_x)) # 不是从字节转成对象,是从字符串转,所以需要这样.
+    
+

funboost/factories/consumer_factory.py

@@ -19,7 +19,7 @@ def get_consumer(boost_params: BoosterParams) -> AbstractConsumer:
     regist_to_funboost(boost_params.broker_kind)  # 动态注册中间件到框架是为了延迟导入，用户没安装不需要的第三方包不报错。
 
     if boost_params.broker_kind not in broker_kind__publsiher_consumer_type_map:
-        raise ValueError(f'设置的中间件种类数字不正确,你设置的值是 {boost_params.broker_kind} ')
+        raise ValueError(f'设置的中间件种类不正确,你设置的值是 {boost_params.broker_kind} ')
     consumer_cls = broker_kind__publsiher_consumer_type_map[boost_params.broker_kind][1]
     if not boost_params.consumer_override_cls:
         return consumer_cls(boost_params)

funboost/factories/publisher_factotry.py

@@ -31,7 +31,7 @@ def get_publisher(publisher_params: PublisherParams) -> AbstractPublisher:
     broker_kind = publisher_params.broker_kind
     regist_to_funboost(broker_kind)  # 动态注册中间件到框架是为了延迟导入，用户没安装不需要的第三方包不报错。
     if broker_kind not in broker_kind__publsiher_consumer_type_map:
-        raise ValueError(f'设置的中间件种类数字不正确,你设置的值是 {broker_kind} ')
+        raise ValueError(f'设置的中间件种类不正确,你设置的值是 {broker_kind} ')
     publisher_cls = broker_kind__publsiher_consumer_type_map[broker_kind][0]
     if not publisher_params.publisher_override_cls:
         return publisher_cls(publisher_params)

funboost/funboost_config_deafult.py

@@ -44,7 +44,8 @@ class BrokerConnConfig(DataClassBase):
     REDIS_PORT = 6379
     REDIS_DB = 7  # redis消息队列所在db，请不要在这个db放太多其他键值对，以及方便你自己可视化查看你的redis db，框架里面有的功能会scan扫描unacked的键名，使用单独的db。
     REDIS_DB_FILTER_AND_RPC_RESULT = 8  # 如果函数做任务参数过滤 或者使用rpc获取结果，使用这个db，因为这个db的键值对多，和redis消息队列db分开
-    REDIS_URL = f'redis://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+    REDIS_SSL = False # 是否使用ssl加密,默认是False
+    REDIS_URL = f'{"rediss" if REDIS_SSL else "redis"}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
 
     NSQD_TCP_ADDRESSES = ['127.0.0.1:4150']
     NSQD_HTTP_CLIENT_HOST = '127.0.0.1'
@@ -79,7 +80,7 @@ class BrokerConnConfig(DataClassBase):
     MQTT_TCP_PORT = 1883
 
     HTTPSQS_HOST = '127.0.0.1'
-    HTTPSQS_PORT = '1218'
+    HTTPSQS_PORT = 1218
     HTTPSQS_AUTH = '123456'
 
     NATS_URL = 'nats://192.168.6.134:4222'

funboost/publishers/base_publisher.py

@@ -10,6 +10,7 @@ import atexit
 import json
 import logging
 import multiprocessing
+from re import S
 import sys
 import threading
 import time
@@ -26,7 +27,7 @@ from funboost.core.loggers import develop_logger
 # from nb_log import LoggerLevelSetterMixin, LoggerMixin
 from funboost.core.loggers import LoggerLevelSetterMixin, FunboostFileLoggerMixin, get_logger
 from funboost.core.msg_result_getter import AsyncResult, AioAsyncResult
-from funboost.core.serialization import Serialization
+from funboost.core.serialization import PickleHelper, Serialization
 from funboost.core.task_id_logger import TaskIdLogger
 from funboost.utils import decorators
 from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
@@ -212,7 +213,20 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         msg = copy.deepcopy(msg)  # 字典是可变对象,不要改变影响用户自身的传参字典. 用户可能继续使用这个传参字典.
         msg, msg_function_kw, extra_params, task_id = self._convert_msg(msg, task_id, priority_control_config)
         t_start = time.time()
-        msg_json = Serialization.to_json_str(msg)
+
+        can_not_json_serializable_keys = Serialization.find_can_not_json_serializable_keys(msg)
+        if can_not_json_serializable_keys:
+            pass
+            self.logger.warning(f'msg 中包含不能序列化的键: {can_not_json_serializable_keys}')
+            # raise ValueError(f'msg 中包含不能序列化的键: {can_not_json_serializable_keys}')
+            new_msg = copy.deepcopy(Serialization.to_dict(msg))
+            for key in can_not_json_serializable_keys:
+                new_msg[key] = PickleHelper.to_str(new_msg[key])
+            new_msg['extra']['can_not_json_serializable_keys'] = can_not_json_serializable_keys
+            msg_json = Serialization.to_json_str(new_msg)
+        else:
+            msg_json = Serialization.to_json_str(msg)
+        # print(msg_json)
         decorators.handle_exception(retry_times=10, is_throw_error=True, time_sleep=0.1)(
             self.concrete_realization_of_publish)(msg_json)
 

funboost/publishers/http_publisher.py

@@ -14,9 +14,15 @@ class HTTPPublisher(AbstractPublisher, ):
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
         self._http = PoolManager(10)
+        self._ip = self.publisher_params.broker_exclusive_config['host']
+        self._port = self.publisher_params.broker_exclusive_config['port']
+        self._ip_port_str = f'{self._ip}:{self._port}'
+        if self._port is None:
+            raise ValueError('please specify port')
+
 
     def concrete_realization_of_publish(self, msg):
-        url = self.queue_name + '/queue'
+        url = self._ip_port_str + '/queue'
         self._http.request('post', url, fields={'msg': msg})
 
     def clear(self):

funboost/publishers/tcp_publisher.py

@@ -10,24 +10,26 @@ class TCPPublisher(AbstractPublisher, ):
     使用tcp作为中间件,不支持持久化，支持分布式
     """
 
-    BUFSIZE = 10240
-
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        """ tcp为消息队列中间件 时候 queue_name 要设置为例如  127.0.0.1:5689"""
-        pass
+        self._bufsize = self.publisher_params.broker_exclusive_config['bufsize']
 
     # noinspection PyAttributeOutsideInit
     def concrete_realization_of_publish(self, msg):
         if not hasattr(self, '_tcp_cli_sock'):
-            ip__port_str = self.queue_name.split(':')
-            ip_port = (ip__port_str[0], int(ip__port_str[1]))
+            # ip__port_str = self.queue_name.split(':')
+            # ip_port = (ip__port_str[0], int(ip__port_str[1]))
+            self._ip = self.publisher_params.broker_exclusive_config['host']
+            self._port = self.publisher_params.broker_exclusive_config['port']
+            self.__ip_port = (self._ip, self._port)
+            if self._port is None:
+                raise ValueError('please specify port')
             tcp_cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            tcp_cli_sock.connect(ip_port)
+            tcp_cli_sock.connect(self.__ip_port)
             self._tcp_cli_sock = tcp_cli_sock
 
         self._tcp_cli_sock.send(msg.encode())
-        self._tcp_cli_sock.recv(self.BUFSIZE)
+        self._tcp_cli_sock.recv(self._bufsize)
 
     def clear(self):
         pass  # udp没有保存消息

funboost/publishers/udp_publisher.py

@@ -10,19 +10,21 @@ class UDPPublisher(AbstractPublisher, ):
     使用udp作为中间件,不支持持久化，支持分布式
     """
 
-    BUFSIZE = 10240
-
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        """ udp为消息队列中间件 时候 queue_name 要设置为例如  127.0.0.1:5689"""
+        self._bufsize = self.publisher_params.broker_exclusive_config['bufsize']
         self.__udp_client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        ip__port_str = self.queue_name.split(':')
-        self.__ip_port = (ip__port_str[0], int(ip__port_str[1]))
+        self._ip = self.publisher_params.broker_exclusive_config['host']
+        self._port = self.publisher_params.broker_exclusive_config['port']
+        self.__ip_port = (self._ip, self._port)
+        if self._port is None:
+            raise ValueError('please specify port')
         self.__udp_client.connect(self.__ip_port)
 
+    # noinspection PyAttributeOutsideInit
     def concrete_realization_of_publish(self, msg):
         self.__udp_client.send(msg.encode('utf-8'), )
-        self.__udp_client.recv(self.BUFSIZE)
+        self.__udp_client.recv(self._bufsize)
 
     def clear(self):
         pass  # udp没有保存消息

funboost/publishers/zeromq_publisher.py

@@ -10,9 +10,13 @@ class ZeroMqPublisher(AbstractPublisher):
     zeromq 中间件的发布者，zeromq基于socket代码，不会持久化，且不需要安装软件。
     """
     def custom_init(self):
+        self._port = self.publisher_params.broker_exclusive_config['port']
+        if self._port is None:
+            raise ValueError('please specify port')
+
         context = ZmqImporter().zmq.Context()
         socket = context.socket(ZmqImporter().zmq.REQ)
-        socket.connect(f"tcp://localhost:{int(self._queue_name)}")
+        socket.connect(f"tcp://localhost:{int(self._port)}")
         self.socket =socket
         self.logger.warning('框架使用 zeromq 中间件方式，必须先启动消费者(消费者会顺便启动broker) ,只有启动了服务端才能发布任务')
 

funboost/timing_job/apscheduler_use_redis_store.py

@@ -1,5 +1,5 @@
 from apscheduler.jobstores.redis import RedisJobStore
-from funboost.utils.redis_manager import RedisMixin
+from funboost.utils.redis_manager import RedisMixin,get_redis_conn_kwargs
 
 from funboost.timing_job import FunboostBackgroundScheduler
 from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
@@ -41,6 +41,21 @@ class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundSc
     #     return 0.1
 
     def _process_jobs(self):
+        """
+        funboost 的做法是 在任务取出阶段就加锁，从根本上防止了重复执行。
+        这个很关键，防止多个apscheduler 实例同时扫描取出同一个定时任务，间接导致重复执行,
+        在apscheduler 3.xx版本这样写来防止多个apscheduler实例 重复执行定时任务的问题,简直是神操作.
+
+        _process_jobs 功能是扫描取出需要运行的定时任务,而不是直接运行定时任务
+        只要扫描取出任务不会取出相同的任务,就间接的决定了不可能重复执行相同的定时任务了.
+        
+
+        不要以为随便在你自己的消费函数加个redis分布式锁就不会重复执行任务了,redis分布式锁是解决相同代码块不会并发执行,而不是解决重复执行.
+        但funboost是神级别骚操作,把分布式锁加到_process_jobs里面,
+        _process_jobs是获取一个即将运行的定时任务,是扫描并删除这个即将运行的定时任务,
+        所以这里加分布式锁能间接解决不重复运行定时任务,一旦任务被取出，就会从 jobstore 中删除,其他实例就无法再取到这个任务了.
+
+        """
         if self.process_jobs_redis_lock_key is None:
             raise ValueError('process_jobs_redis_lock_key is not set')
         with RedisDistributedBlockLockContextManager(RedisMixin().redis_db_frame, self.process_jobs_redis_lock_key, ):
@@ -48,9 +63,8 @@ class FunboostBackgroundSchedulerProcessJobsWithinRedisLock(FunboostBackgroundSc
 
 
 jobstores = {
-    "default": RedisJobStore(db=BrokerConnConfig.REDIS_DB, host=BrokerConnConfig.REDIS_HOST,
-                             port=BrokerConnConfig.REDIS_PORT, password=BrokerConnConfig.REDIS_PASSWORD,
-                             username=BrokerConnConfig.REDIS_USERNAME, jobs_key='funboost.apscheduler.jobs',run_times_key="funboost.apscheduler.run_times")
+    "default": RedisJobStore(**get_redis_conn_kwargs(),
+                             jobs_key='funboost.apscheduler.jobs',run_times_key="funboost.apscheduler.run_times")
 }
 
 """

funboost/timing_job/timing_push.py

@@ -146,7 +146,7 @@ if __name__ == '__main__':
         id='interval_job1'
     )
 
-    # 方式3：使用cron表达式定时执行
+    # 方式3：使用cron表达式定时执行,周期运行
     ApsJobAdder(sum_two_numbers, job_store_kind='redis').add_push_job(
         trigger='cron',
         day_of_week='*',
@@ -156,5 +156,7 @@ if __name__ == '__main__':
         kwargs={"x": 50, "y": 60},
         replace_existing=True,
         id='cron_job1')
+    
+
 
     ctrl_c_recv() # 启动了守护线程的定时器，一定要阻止主线程退出。 你可以代码最末尾加这个 ctrl_c_recv() 或者加个 while 1:time.sleep(10)

funboost/utils/ctrl_c_end.py

@@ -16,7 +16,7 @@ def ctrl_c_recv():
     apscheduler background 类型必须有主线程在运行，否则会很快结束。所以需要阻止主线程退出。
     在代码最最末尾加上 ctrl_c_recv() 就可以阻止主线程退出。
     
-    你也可以直接在你的启动脚本的最末尾加上：
+    你也可以不用ctrl_c_recv(),  直接在你的启动脚本文件的最末尾加上：
     while 1:
         time.sleep(100) 
     来达到阻止主线程退出的目的。

funboost/utils/redis_manager.py

@@ -13,7 +13,7 @@ from funboost.utils import decorators
 
 def get_redis_conn_kwargs():
     return {'host': BrokerConnConfig.REDIS_HOST, 'port': BrokerConnConfig.REDIS_PORT,
-            'username': BrokerConnConfig.REDIS_USERNAME,
+            'username': BrokerConnConfig.REDIS_USERNAME,'ssl' : BrokerConnConfig.REDIS_SSL,
             'password': BrokerConnConfig.REDIS_PASSWORD, 'db': BrokerConnConfig.REDIS_DB}
 
 
@@ -26,11 +26,13 @@ def _get_redis_conn_kwargs_by_db(db):
 class RedisManager(object):
     _redis_db__conn_map = {}
 
-    def __init__(self, host='127.0.0.1', port=6379, db=0, username='', password=''):
-        self._key = (host, port, db, username, password,)
+    def __init__(self, host='127.0.0.1', port=6379, db=0, username='', password='',ssl=False):
+        self._key = (host, port, db, username, password,ssl)
         if self._key not in self.__class__._redis_db__conn_map:
             self.__class__._redis_db__conn_map[self._key] = redis5.Redis(host=host, port=port, db=db, username=username,
-                                                                         password=password, max_connections=1000, decode_responses=True)
+                                                                         password=password, max_connections=1000,
+                                                                         ssl=ssl,
+                                                                         decode_responses=True)
         self.redis = self.__class__._redis_db__conn_map[self._key]
 
     def get_redis(self) -> redis5.Redis: