funboost 49.6__py3-none-any.whl → 49.8__py3-none-any.whl

funboost/core/serialization.py

@@ -1,6 +1,9 @@
 import typing
-
+import json
 import orjson
+import pickle
+import ast
+
 class Serialization:
     @staticmethod
     def to_json_str(dic:typing.Union[dict,str]):
@@ -14,3 +17,27 @@ class Serialization:
         if isinstance(strx,dict):
             return strx
         return orjson.loads(strx)
+    
+    @staticmethod
+    def find_can_not_json_serializable_keys(dic:dict)->typing.List[str]:
+        can_not_json_serializable_keys = []
+        dic = Serialization.to_dict(dic)
+        for k,v in dic.items():
+            if not isinstance(v,str):
+                try:
+                    json.dumps(v)
+                except:
+                    can_not_json_serializable_keys.append(k)
+        return can_not_json_serializable_keys
+    
+
+class PickleHelper:
+    @staticmethod
+    def to_str(obj_x:typing.Any):
+        return str(pickle.dumps(obj_x)) # 对象pickle,转成字符串
+    
+    @staticmethod
+    def to_obj(str_x:str):
+        return pickle.loads(ast.literal_eval(str_x)) # 不是从字节转成对象,是从字符串转,所以需要这样.
+    
+

funboost/factories/consumer_factory.py

@@ -19,7 +19,7 @@ def get_consumer(boost_params: BoosterParams) -> AbstractConsumer:
     regist_to_funboost(boost_params.broker_kind)  # 动态注册中间件到框架是为了延迟导入，用户没安装不需要的第三方包不报错。
 
     if boost_params.broker_kind not in broker_kind__publsiher_consumer_type_map:
-        raise ValueError(f'设置的中间件种类数字不正确,你设置的值是 {boost_params.broker_kind} ')
+        raise ValueError(f'设置的中间件种类不正确,你设置的值是 {boost_params.broker_kind} ')
     consumer_cls = broker_kind__publsiher_consumer_type_map[boost_params.broker_kind][1]
     if not boost_params.consumer_override_cls:
         return consumer_cls(boost_params)

funboost/factories/publisher_factotry.py

@@ -31,7 +31,7 @@ def get_publisher(publisher_params: PublisherParams) -> AbstractPublisher:
     broker_kind = publisher_params.broker_kind
     regist_to_funboost(broker_kind)  # 动态注册中间件到框架是为了延迟导入，用户没安装不需要的第三方包不报错。
     if broker_kind not in broker_kind__publsiher_consumer_type_map:
-        raise ValueError(f'设置的中间件种类数字不正确,你设置的值是 {broker_kind} ')
+        raise ValueError(f'设置的中间件种类不正确,你设置的值是 {broker_kind} ')
     publisher_cls = broker_kind__publsiher_consumer_type_map[broker_kind][0]
     if not publisher_params.publisher_override_cls:
         return publisher_cls(publisher_params)

funboost/funboost_config_deafult.py

@@ -44,7 +44,7 @@ class BrokerConnConfig(DataClassBase):
     REDIS_PORT = 6379
     REDIS_DB = 7  # redis消息队列所在db，请不要在这个db放太多其他键值对，以及方便你自己可视化查看你的redis db，框架里面有的功能会scan扫描unacked的键名，使用单独的db。
     REDIS_DB_FILTER_AND_RPC_RESULT = 8  # 如果函数做任务参数过滤 或者使用rpc获取结果，使用这个db，因为这个db的键值对多，和redis消息队列db分开
-    REDIS_SSL = False
+    REDIS_SSL = False # 是否使用ssl加密,默认是False
     REDIS_URL = f'{"rediss" if REDIS_SSL else "redis"}://{REDIS_USERNAME}:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
 
     NSQD_TCP_ADDRESSES = ['127.0.0.1:4150']
@@ -80,7 +80,7 @@ class BrokerConnConfig(DataClassBase):
     MQTT_TCP_PORT = 1883
 
     HTTPSQS_HOST = '127.0.0.1'
-    HTTPSQS_PORT = '1218'
+    HTTPSQS_PORT = 1218
     HTTPSQS_AUTH = '123456'
 
     NATS_URL = 'nats://192.168.6.134:4222'

funboost/publishers/base_publisher.py

@@ -10,6 +10,7 @@ import atexit
 import json
 import logging
 import multiprocessing
+from re import S
 import sys
 import threading
 import time
@@ -26,7 +27,7 @@ from funboost.core.loggers import develop_logger
 # from nb_log import LoggerLevelSetterMixin, LoggerMixin
 from funboost.core.loggers import LoggerLevelSetterMixin, FunboostFileLoggerMixin, get_logger
 from funboost.core.msg_result_getter import AsyncResult, AioAsyncResult
-from funboost.core.serialization import Serialization
+from funboost.core.serialization import PickleHelper, Serialization
 from funboost.core.task_id_logger import TaskIdLogger
 from funboost.utils import decorators
 from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
@@ -212,7 +213,20 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
         msg = copy.deepcopy(msg)  # 字典是可变对象,不要改变影响用户自身的传参字典. 用户可能继续使用这个传参字典.
         msg, msg_function_kw, extra_params, task_id = self._convert_msg(msg, task_id, priority_control_config)
         t_start = time.time()
-        msg_json = Serialization.to_json_str(msg)
+
+        can_not_json_serializable_keys = Serialization.find_can_not_json_serializable_keys(msg)
+        if can_not_json_serializable_keys:
+            pass
+            self.logger.warning(f'msg 中包含不能序列化的键: {can_not_json_serializable_keys}')
+            # raise ValueError(f'msg 中包含不能序列化的键: {can_not_json_serializable_keys}')
+            new_msg = copy.deepcopy(Serialization.to_dict(msg))
+            for key in can_not_json_serializable_keys:
+                new_msg[key] = PickleHelper.to_str(new_msg[key])
+            new_msg['extra']['can_not_json_serializable_keys'] = can_not_json_serializable_keys
+            msg_json = Serialization.to_json_str(new_msg)
+        else:
+            msg_json = Serialization.to_json_str(msg)
+        # print(msg_json)
         decorators.handle_exception(retry_times=10, is_throw_error=True, time_sleep=0.1)(
             self.concrete_realization_of_publish)(msg_json)
 

funboost/publishers/http_publisher.py

@@ -14,9 +14,15 @@ class HTTPPublisher(AbstractPublisher, ):
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
         self._http = PoolManager(10)
+        self._ip = self.publisher_params.broker_exclusive_config['host']
+        self._port = self.publisher_params.broker_exclusive_config['port']
+        self._ip_port_str = f'{self._ip}:{self._port}'
+        if self._port is None:
+            raise ValueError('please specify port')
+
 
     def concrete_realization_of_publish(self, msg):
-        url = self.queue_name + '/queue'
+        url = self._ip_port_str + '/queue'
         self._http.request('post', url, fields={'msg': msg})
 
     def clear(self):

funboost/publishers/tcp_publisher.py

@@ -10,24 +10,26 @@ class TCPPublisher(AbstractPublisher, ):
     使用tcp作为中间件,不支持持久化，支持分布式
     """
 
-    BUFSIZE = 10240
-
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        """ tcp为消息队列中间件 时候 queue_name 要设置为例如  127.0.0.1:5689"""
-        pass
+        self._bufsize = self.publisher_params.broker_exclusive_config['bufsize']
 
     # noinspection PyAttributeOutsideInit
     def concrete_realization_of_publish(self, msg):
         if not hasattr(self, '_tcp_cli_sock'):
-            ip__port_str = self.queue_name.split(':')
-            ip_port = (ip__port_str[0], int(ip__port_str[1]))
+            # ip__port_str = self.queue_name.split(':')
+            # ip_port = (ip__port_str[0], int(ip__port_str[1]))
+            self._ip = self.publisher_params.broker_exclusive_config['host']
+            self._port = self.publisher_params.broker_exclusive_config['port']
+            self.__ip_port = (self._ip, self._port)
+            if self._port is None:
+                raise ValueError('please specify port')
             tcp_cli_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-            tcp_cli_sock.connect(ip_port)
+            tcp_cli_sock.connect(self.__ip_port)
             self._tcp_cli_sock = tcp_cli_sock
 
         self._tcp_cli_sock.send(msg.encode())
-        self._tcp_cli_sock.recv(self.BUFSIZE)
+        self._tcp_cli_sock.recv(self._bufsize)
 
     def clear(self):
         pass  # udp没有保存消息

funboost/publishers/udp_publisher.py

@@ -10,19 +10,21 @@ class UDPPublisher(AbstractPublisher, ):
     使用udp作为中间件,不支持持久化，支持分布式
     """
 
-    BUFSIZE = 10240
-
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        """ udp为消息队列中间件 时候 queue_name 要设置为例如  127.0.0.1:5689"""
+        self._bufsize = self.publisher_params.broker_exclusive_config['bufsize']
         self.__udp_client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
-        ip__port_str = self.queue_name.split(':')
-        self.__ip_port = (ip__port_str[0], int(ip__port_str[1]))
+        self._ip = self.publisher_params.broker_exclusive_config['host']
+        self._port = self.publisher_params.broker_exclusive_config['port']
+        self.__ip_port = (self._ip, self._port)
+        if self._port is None:
+            raise ValueError('please specify port')
         self.__udp_client.connect(self.__ip_port)
 
+    # noinspection PyAttributeOutsideInit
     def concrete_realization_of_publish(self, msg):
         self.__udp_client.send(msg.encode('utf-8'), )
-        self.__udp_client.recv(self.BUFSIZE)
+        self.__udp_client.recv(self._bufsize)
 
     def clear(self):
         pass  # udp没有保存消息

funboost/publishers/zeromq_publisher.py

@@ -10,9 +10,13 @@ class ZeroMqPublisher(AbstractPublisher):
     zeromq 中间件的发布者，zeromq基于socket代码，不会持久化，且不需要安装软件。
     """
     def custom_init(self):
+        self._port = self.publisher_params.broker_exclusive_config['port']
+        if self._port is None:
+            raise ValueError('please specify port')
+
         context = ZmqImporter().zmq.Context()
         socket = context.socket(ZmqImporter().zmq.REQ)
-        socket.connect(f"tcp://localhost:{int(self._queue_name)}")
+        socket.connect(f"tcp://localhost:{int(self._port)}")
         self.socket =socket
         self.logger.warning('框架使用 zeromq 中间件方式，必须先启动消费者(消费者会顺便启动broker) ,只有启动了服务端才能发布任务')
 

funboost/timing_job/timing_push.py

@@ -146,7 +146,7 @@ if __name__ == '__main__':
         id='interval_job1'
     )
 
-    # 方式3：使用cron表达式定时执行
+    # 方式3：使用cron表达式定时执行,周期运行
     ApsJobAdder(sum_two_numbers, job_store_kind='redis').add_push_job(
         trigger='cron',
         day_of_week='*',
@@ -156,5 +156,7 @@ if __name__ == '__main__':
         kwargs={"x": 50, "y": 60},
         replace_existing=True,
         id='cron_job1')
+    
+
 
     ctrl_c_recv() # 启动了守护线程的定时器，一定要阻止主线程退出。 你可以代码最末尾加这个 ctrl_c_recv() 或者加个 while 1:time.sleep(10)