funboost 44.0__py3-none-any.whl → 44.2__py3-none-any.whl

funboost/publishers/base_publisher.py

@@ -9,17 +9,15 @@ import json
 import logging
 import multiprocessing
 import threading
-import uuid
 import time
 import typing
 from functools import wraps
 from threading import Lock
-import datetime
 import amqpstorm
 
 import nb_log
 from funboost.core.func_params_model import PublisherParams, PriorityConsumingControlConfig
-from funboost.core.helper_funs import generate_task_id
+from funboost.core.helper_funs import MsgGenerater
 from funboost.core.loggers import develop_logger
 
 from pikav1.exceptions import AMQPError as PikaAMQPError
@@ -28,7 +26,7 @@ from pikav1.exceptions import AMQPError as PikaAMQPError
 from funboost.core.loggers import LoggerLevelSetterMixin, FunboostFileLoggerMixin, get_logger
 from funboost.core.msg_result_getter import AsyncResult, AioAsyncResult
 from funboost.core.task_id_logger import TaskIdLogger
-from funboost.utils import decorators, time_util
+from funboost.utils import decorators
 from funboost.funboost_config_deafult import BrokerConnConfig, FunboostCommonConfig
 
 RedisAsyncResult = AsyncResult  # 别名
@@ -191,9 +189,8 @@ class AbstractPublisher(LoggerLevelSetterMixin, metaclass=abc.ABCMeta, ):
             raw_extra = msg['extra']
         if self.publish_params_checker and self.publisher_params.should_check_publish_func_params:
             self.publish_params_checker.check_params(msg_function_kw)
-        task_id = task_id or generate_task_id(self._queue_name)
-        extra_params = {'task_id': task_id, 'publish_time': round(time.time(), 4),
-                        'publish_time_format': time.strftime('%Y-%m-%d %H:%M:%S')}
+        task_id = task_id or MsgGenerater.generate_task_id(self._queue_name)
+        extra_params = MsgGenerater.generate_pulish_time_and_task_id(self._queue_name, task_id=task_id)
         if priority_control_config:
             extra_params.update(priority_control_config.dict(exclude_none=True))
         extra_params.update(raw_extra)

funboost/publishers/confluent_kafka_publisher.py

@@ -4,6 +4,8 @@
 
 import os
 
+from funboost.core.lazy_impoter import KafkaPythonImporter
+
 if os.name == 'nt':
     """
     为了保险起见，这样做一下,设置一下path，否则anaconda安装的python可能出现 ImportError: DLL load failed while importing cimpl: 找不到指定的模块。
@@ -24,12 +26,7 @@ if os.name == 'nt':
 
 import atexit
 import time
-# noinspection PyPackageRequirements
-from kafka import KafkaProducer, KafkaAdminClient
-# noinspection PyPackageRequirements
-from kafka.admin import NewTopic
-# noinspection PyPackageRequirements
-from kafka.errors import TopicAlreadyExistsError
+
 from confluent_kafka import Producer as ConfluentProducer
 from funboost.funboost_config_deafult import BrokerConnConfig
 from funboost.publishers.base_publisher import AbstractPublisher
@@ -45,10 +42,10 @@ class ConfluentKafkaPublisher(AbstractPublisher, ):
 
         # self._producer = KafkaProducer(bootstrap_servers=funboost_config_deafult.KAFKA_BOOTSTRAP_SERVERS)
         try:
-            admin_client = KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client = KafkaPythonImporter().KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
         except BaseException as e:
             self.logger.exception(e)
@@ -92,10 +89,10 @@ class SaslPlainKafkaPublisher(ConfluentKafkaPublisher):
     def custom_init(self):
         # self._producer = KafkaProducer(bootstrap_servers=funboost_config_deafult.KAFKA_BOOTSTRAP_SERVERS)
         try:
-            admin_client = KafkaAdminClient(**BrokerConnConfig.KFFKA_SASL_CONFIG)
-            admin_client.create_topics([NewTopic(self._queue_name, 10, 1)])
+            admin_client = KafkaPythonImporter().KafkaAdminClient(**BrokerConnConfig.KFFKA_SASL_CONFIG)
+            admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 1)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
         except BaseException as e:
             self.logger.exception(e)

funboost/publishers/kafka_publisher.py

@@ -5,13 +5,7 @@
 # noinspection PyPackageRequirements
 import atexit
 
-# noinspection PyPackageRequirements
-from kafka import KafkaProducer, KafkaAdminClient
-# noinspection PyPackageRequirements
-from kafka.admin import NewTopic
-# noinspection PyPackageRequirements
-from kafka.errors import TopicAlreadyExistsError
-
+from funboost.core.lazy_impoter import KafkaPythonImporter
 from funboost.funboost_config_deafult import BrokerConnConfig
 from funboost.publishers.base_publisher import AbstractPublisher
 
@@ -23,12 +17,12 @@ class KafkaPublisher(AbstractPublisher, ):
 
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        self._producer = KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
-        self._admin_client = KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+        self._producer = KafkaPythonImporter().KafkaProducer(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
+        self._admin_client = KafkaPythonImporter().KafkaAdminClient(bootstrap_servers=BrokerConnConfig.KAFKA_BOOTSTRAP_SERVERS)
         try:
-            self._admin_client.create_topics([NewTopic(self._queue_name, 10, 2)])
+            self._admin_client.create_topics([KafkaPythonImporter().NewTopic(self._queue_name, 10, 2)])
             # admin_client.create_partitions({self._queue_name: NewPartitions(total_count=16)})
-        except TopicAlreadyExistsError:
+        except KafkaPythonImporter().TopicAlreadyExistsError:
             pass
         except BaseException as e:
             self.logger.exception(e)

funboost/publishers/mqtt_publisher.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # @Author  : ydf
 # @Time    : 2022/8/8 0008 12:12
+from funboost.core.lazy_impoter import PahoMqttImporter
 from funboost.publishers.base_publisher import AbstractPublisher
 from funboost.funboost_config_deafult import BrokerConnConfig
 
@@ -53,7 +54,7 @@ client.loop_forever() # 保持连接
 著作权归作者所有。商业转载请联系作者获得授权，非商业转载请注明出处。
 """
 
-import paho.mqtt.client as mqtt
+# import paho.mqtt.client as mqtt
 
 
 # def on_connect(client, userdata, flags, rc):
@@ -71,7 +72,7 @@ class MqttPublisher(AbstractPublisher, ):
 
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        client = mqtt.Client()
+        client = PahoMqttImporter().mqtt.Client()
         # client.username_pw_set('admin', password='public')
         client.on_connect = self._on_connect
         client.on_socket_close = self._on_socket_close

funboost/publishers/nats_publisher.py

@@ -1,4 +1,4 @@
-from pynats import NATSClient  # noqa
+from funboost.core.lazy_impoter import NatsImporter
 from funboost.publishers.base_publisher import AbstractPublisher
 from funboost.funboost_config_deafult import BrokerConnConfig
 
@@ -10,7 +10,7 @@ class NatsPublisher(AbstractPublisher, ):
 
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        self.nats_client = NATSClient(BrokerConnConfig.NATS_URL)
+        self.nats_client = NatsImporter().NATSClient(BrokerConnConfig.NATS_URL)
         self.nats_client.connect()
 
     def concrete_realization_of_publish(self, msg):

funboost/publishers/nsq_publisher.py

@@ -1,9 +1,7 @@
 # -*- coding: utf-8 -*-
 # @Author  : ydf
 # @Time    : 2022/8/19 0008 12:12
-
-from gnsq import Producer, NsqdHTTPClient
-from gnsq.errors import NSQHttpError
+from funboost.core.lazy_impoter import GnsqImporter
 from funboost.publishers.base_publisher import AbstractPublisher
 from funboost.funboost_config_deafult import BrokerConnConfig
 
@@ -15,8 +13,8 @@ class NsqPublisher(AbstractPublisher, ):
 
     # noinspection PyAttributeOutsideInit
     def custom_init(self):
-        self._nsqd_cleint = NsqdHTTPClient(BrokerConnConfig.NSQD_HTTP_CLIENT_HOST, BrokerConnConfig.NSQD_HTTP_CLIENT_PORT)
-        self._producer = Producer(BrokerConnConfig.NSQD_TCP_ADDRESSES)
+        self._nsqd_cleint = GnsqImporter().NsqdHTTPClient(BrokerConnConfig.NSQD_HTTP_CLIENT_HOST, BrokerConnConfig.NSQD_HTTP_CLIENT_PORT)
+        self._producer = GnsqImporter().Producer(BrokerConnConfig.NSQD_TCP_ADDRESSES)
         self._producer.start()
 
     def concrete_realization_of_publish(self, msg):
@@ -26,7 +24,7 @@ class NsqPublisher(AbstractPublisher, ):
     def clear(self):
         try:
             self._nsqd_cleint.empty_topic(self._queue_name)
-        except NSQHttpError as e:
+        except GnsqImporter().NSQHttpError as e:
             self.logger.exception(e)  # 不能清除一个不存在的topoc会报错，和其他消息队列中间件不同。
         self.logger.warning(f'清除 {self._queue_name} topic中的消息成功')
 

funboost/publishers/zeromq_publisher.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 # @Author  : ydf
-import zmq
+from funboost.core.lazy_impoter import ZmqImporter
 from funboost.publishers.base_publisher import AbstractPublisher
 
 
@@ -10,8 +10,8 @@ class ZeroMqPublisher(AbstractPublisher):
     zeromq 中间件的发布者，zeromq基于socket代码，不会持久化，且不需要安装软件。
     """
     def custom_init(self):
-        context = zmq.Context()
-        socket = context.socket(zmq.REQ)
+        context = ZmqImporter().zmq.Context()
+        socket = context.socket(ZmqImporter().zmq.REQ)
         socket.connect(f"tcp://localhost:{int(self._queue_name)}")
         self.socket =socket
         self.logger.warning('框架使用 zeromq 中间件方式，必须先启动消费者(消费者会顺便启动broker) ,只有启动了服务端才能发布任务')

funboost/set_frame_config.py

@@ -12,7 +12,7 @@ import json
 from pathlib import Path
 from shutil import copyfile
 
-from funboost.core.helper_funs import _try_get_user_funboost_common_config
+from funboost.core.funboost_config_getter import _try_get_user_funboost_common_config
 from funboost.core.loggers import flogger, get_funboost_file_logger, logger_prompt
 from nb_log import nb_print, stderr_write, stdout_write
 from nb_log.monkey_print import is_main_process, only_print_on_main_process

funboost/utils/bulk_operation.py

@@ -9,7 +9,7 @@
 import atexit
 import re
 import os
-from elasticsearch import helpers
+# from elasticsearch import helpers
 from threading import Thread
 from typing import Union
 import abc
@@ -20,6 +20,7 @@ import unittest
 from pymongo import UpdateOne, InsertOne, UpdateMany, collection, MongoClient
 import redis
 
+from funboost.core.lazy_impoter import ElasticsearchImporter
 from funboost.utils.redis_manager import RedisMixin
 from funboost.utils.time_util import DatetimeConverter
 from funboost.utils import LoggerMixin, decorators
@@ -153,7 +154,7 @@ class ElasticBulkHelper(BaseBulkHelper):
                     break
             if request_list:
                 # self.base_object.bulk_write(request_list, ordered=False)
-                helpers.bulk(self.base_object, request_list)
+                ElasticsearchImporter().helpers.bulk(self.base_object, request_list)
             if self._is_print_log:
                 self.logger.info(f'【{self.base_object}】  批量插入的任务数量是 {count} 消耗的时间是 {round(time.time() - t_start, 6)}')
             self._current_time = time.time()

funboost/utils/decorators.py

@@ -6,8 +6,6 @@ import logging
 import random
 import uuid
 from typing import TypeVar
-
-from flask import request as flask_request
 # noinspection PyUnresolvedReferences
 from contextlib import contextmanager
 import functools

funboost/utils/resource_monitoring.py

@@ -7,7 +7,8 @@ import socket
 import sys
 import threading
 import time
-import psutil
+
+from funboost.core.lazy_impoter import PsutilImporter
 from funboost.utils import LoggerLevelSetterMixin, LoggerMixin, decorators
 from funboost.utils.mongo_util import MongoMixin
 
@@ -59,10 +60,10 @@ print(psutil.virtual_memory())
 
 class ResourceMonitor(LoggerMixin, LoggerLevelSetterMixin, MongoMixin):
     # ResourceMonitor(is_save_info_to_mongo=True).set_log_level(20).start_build_info_loop_on_daemon_thread(60)
-    cpu_count = psutil.cpu_count()
+    cpu_count = PsutilImporter().psutil.cpu_count()
     host_name = socket.gethostname()
 
-    def __init__(self, process=psutil.Process(), is_save_info_to_mongo=False, mongo_col='default'):
+    def __init__(self, process=PsutilImporter().psutil.Process(), is_save_info_to_mongo=False, mongo_col='default'):
         self.process = process
         self.logger.setLevel(20)
         self.all_info = {}
@@ -84,22 +85,22 @@ class ResourceMonitor(LoggerMixin, LoggerLevelSetterMixin, MongoMixin):
         return result
 
     def get_os_cpu_percpu(self):
-        result = psutil.cpu_percent(1, percpu=True)
+        result = PsutilImporter().psutil.cpu_percent(1, percpu=True)
         self.logger.debug(result)
         return result
 
     def get_os_cpu_totalcpu(self):
-        result = round(psutil.cpu_percent(1, percpu=False) * self.cpu_count, 2)
+        result = round(PsutilImporter().psutil.cpu_percent(1, percpu=False) * self.cpu_count, 2)
         self.logger.debug(result)
         return result
 
     def get_os_cpu_avaragecpu(self):
-        result = psutil.cpu_percent(1, percpu=False)
+        result = PsutilImporter().psutil.cpu_percent(1, percpu=False)
         self.logger.debug(result)
         return result
 
     def get_os_virtual_memory(self) -> dict:
-        memory_tuple = psutil.virtual_memory()
+        memory_tuple = PsutilImporter().psutil.virtual_memory()
         self.logger.debug(memory_tuple)
         return {
             'total': self.divide_1m(memory_tuple[0]),
@@ -108,9 +109,9 @@ class ResourceMonitor(LoggerMixin, LoggerLevelSetterMixin, MongoMixin):
         }
 
     def get_os_net_info(self):
-        result1 = psutil.net_io_counters(pernic=False)
+        result1 = PsutilImporter().psutil.net_io_counters(pernic=False)
         time.sleep(1)
-        result2 = psutil.net_io_counters(pernic=False)
+        result2 = PsutilImporter().psutil.net_io_counters(pernic=False)
         speed_dict = dict()
         speed_dict['up_speed'] = self.divide_1m(result2[0] - result1[0])
         speed_dict['down_speed'] = self.divide_1m(result2[1] - result1[1])