toolkits 0.2.6__tar.gz → 0.2.7__tar.gz

toolkits-0.2.7/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.1
+Name: toolkits
+Version: 0.2.7
+Summary: toolkits for quickly reference
+Author-Email: li_jia_yue <59727816@qq.com>
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: fire>=0.6.0
+Requires-Dist: arrow>=1.3.0
+Requires-Dist: pytest>=8.2.2
+Requires-Dist: icecream>=2.1.3
+Requires-Dist: schedule>=1.2.2
+Requires-Dist: jmespath>=1.0.1
+Requires-Dist: unicodecsv>=0.14.1
+Requires-Dist: log4python>=1.0.11
+Requires-Dist: unipath>=1.1
+Description-Content-Type: text/markdown
+
+# toolkits
+## 迁移PDM项目
+1, Create virtual env 
+pdm venv create
+
+2, Install dependency
+pdm install 
+
+3, basic python libs 
+pdm add arrow toolkits requests dingtalkchatbot exchangelib redis PyYAML sqlalchemy kafka-python-ng urllib3==1.26.15 wheel jmespath log4python PyMySQL pytest crython icecream
+
+## 使用包中的Python脚本 
+### 设置Python环境变量 
+      export PYTHONPATH="/home/pythonDemo/toolkits/src/:$PYTHONPATH"
+### 源码目录中不要有[与包名一致的Python脚本文件] 
+### 直接调用Python脚本 
+      python toolkits/db_query_demo.py --config_path="/home/pythonDemo/toolkits/src/toolkits/config/config_demo.py" worker test_01.txt 

toolkits-0.2.7/README.md

@@ -0,0 +1,17 @@
+# toolkits
+## 迁移PDM项目
+1, Create virtual env 
+pdm venv create
+
+2, Install dependency
+pdm install 
+
+3, basic python libs 
+pdm add arrow toolkits requests dingtalkchatbot exchangelib redis PyYAML sqlalchemy kafka-python-ng urllib3==1.26.15 wheel jmespath log4python PyMySQL pytest crython icecream
+
+## 使用包中的Python脚本 
+### 设置Python环境变量 
+      export PYTHONPATH="/home/pythonDemo/toolkits/src/:$PYTHONPATH"
+### 源码目录中不要有[与包名一致的Python脚本文件] 
+### 直接调用Python脚本 
+      python toolkits/db_query_demo.py --config_path="/home/pythonDemo/toolkits/src/toolkits/config/config_demo.py" worker test_01.txt 

toolkits-0.2.7/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "toolkits"
+version = "0.2.7"
+description = "toolkits for quickly reference"
+authors = [
+    { name = "li_jia_yue", email = "59727816@qq.com" },
+]
+dependencies = [
+    "fire>=0.6.0",
+    "arrow>=1.3.0",
+    "pytest>=8.2.2",
+    "icecream>=2.1.3",
+    "schedule>=1.2.2",
+    "jmespath>=1.0.1",
+    "unicodecsv>=0.14.1",
+    "log4python>=1.0.11",
+    "unipath>=1.1",
+]
+requires-python = ">=3.9"
+readme = "README.md"
+
+[project.license]
+text = "MIT"
+
+[project.scripts]
+toolkits = "toolkits.toolkits_cli:main"
+
+[build-system]
+requires = [
+    "pdm-backend",
+]
+build-backend = "pdm.backend"
+
+[tool.pdm]
+distribution = true

toolkits-0.2.7/src/toolkits/3des/3des.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+import pyDes
+import base64
+
+
+class TripleDesUtils:
+    des_mode = {"CBC": pyDes.CBC, "ECB": pyDes.ECB}
+    des_pad_mode = {"PAD_PKCS5": pyDes.PAD_PKCS5, "PAD_NORMAL": pyDes.PAD_NORMAL}
+
+    def __init__(self, mode, pad_mode, key, iv, pad=None, trans_base64=False):
+        """
+        :param mode: des 加密模式，目前支持 CBC，ECB
+        :param pad_mode: 目前支持 PAD_PKCS5，PAD_NORMAL
+        :param trans_base64: 加密结果是否以 base64 格式输出
+        :param key: 密钥
+        :param iv: 偏移量
+        :param pad:
+        """
+        self.trans_base64 = trans_base64
+        self.k = pyDes.triple_des(key, TripleDesUtils.des_mode.get(mode), iv, pad, TripleDesUtils.des_pad_mode.get(pad_mode))
+
+    def encryption(self, data):
+        """
+        3des 加密
+        说明: 3DES数据块长度为64位，所以IV长度需要为8个字符（ECB模式不用IV），密钥长度为16或24个字符（8个字符以内则结果与DES相同
+        IV与密钥超过长度则截取，不足则在末尾填充'\0'补足
+        :param data: 待加密数据
+        :return:
+        """
+        _encryption_result = self.k.encrypt(data)
+        if self.trans_base64:
+            _encryption_result = self._base64encode(_encryption_result)
+        return _encryption_result
+
+    def decrypt(self, data):
+        """
+        3des 解密
+        :param data: 待解密数据
+        :return:
+        """
+        if self.trans_base64:
+            data = self._base64decode(data)
+        _decrypt_result = self.k.decrypt(data)
+        return _decrypt_result
+
+    @staticmethod
+    def _base64encode(data):
+        """
+        base 64 encode
+        :param data: encode data
+        :return:
+        """
+        try:
+            _b64encode_result = base64.b64encode(data)
+        except Exception as e:
+            raise Exception("base64 encode error:{e}")
+        return _b64encode_result
+
+    @staticmethod
+    def _base64decode(data):
+        """
+        base 64 decode
+        :param data: decode data
+        :return:
+        """
+        try:
+            _b64decode_result = base64.b64decode(data)
+        except Exception as e:
+            raise Exception("base64 decode error:{e}")
+        return _b64decode_result
+
+
+if __name__ == "__main__":
+    test_data = "12345678a"
+    key_a = "uusafeuusafeuusafeuusafe"
+    key_b = "jiayufeuusafeuusafeuusaf"
+    # [12345678a] 3des Result: 2yjtt0Y/c7xEOa9VGetBVA==
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_a, iv="01234567", trans_base64=True)
+    result_a = DesObj.encryption(test_data)
+    print("加密结果: %s" % result_a)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_b, iv="01234567", trans_base64=True)
+    result_b = DesObj.encryption(result_a)
+    print("加密结果: %s" % result_b)
+
+    # result2 = DesObj.decrypt(result)
+    # print("解密结果: %s" % result2)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_b, iv="01234567", trans_base64=True)
+    result_a = DesObj.encryption(test_data)
+    print("加密结果: %s" % result_a)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_a, iv="01234567", trans_base64=True)
+    result_b = DesObj.encryption(result_a)
+    print("加密结果: %s" % result_b)

toolkits-0.2.7/src/toolkits/config/config_demo.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+config_global = {
+    "db_list": {
+        'dev_ops': {
+            'desc': "Home 数据库",
+            'db_name': "dev_ops",
+            'user_name': 'root',
+            'password': '4tKTiDYr81YM',
+            'host': "192.168.100.155",
+            'port': 6606
+        },
+        'online_alarm':  {
+            'desc': "16.15 数据库",
+            'user_name': 'db_admin',
+            'password': '123QWEas!@#',
+            'host': "10.83.16.15",
+            'port': 8012,
+            'db_name': "sec_admin"
+        }
+    },
+    "kafka_list": {
+        "home": {
+            'host': "192.168.100.155",
+            'port': 9092
+        }
+    },
+    "redis_list": {
+        "home": {
+            'password': 's1hKcWqRj9Se',
+            'host': '192.168.100.155',
+            'port': 9379,
+            'db': 1
+        }
+    },
+    "zookeeper_list": {
+        "home": {
+            "bin_path": "/usr/local/dev/kafka/kafka_2.12-2.3.0",
+            "ip": "192.168.100.155",
+            "port": "2184"
+        }
+    }
+}

toolkits-0.2.7/src/toolkits/db_query_demo.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+import json
+import traceback
+
+import fire
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory
+
+from toolkits.libs_core.env_prepare import EnvPrepare
+from toolkits.libs_core.mysql_helper import MysqlHelper
+
+logger = log("DbQueryDemo")
+
+
+class DbQueryDemo:
+    def __init__(self, config_path=None):
+        self.__base_path = get_script_directory()
+        self.__config_path = config_path
+        if config_path is None:
+            if tookitsApp.config_file_path is None:
+                self.__env = EnvPrepare(tookitsApp.app_name)
+                self.__env.check_env()
+                self.__config_path = self.__env.get_config_path("config.py")
+                self.__env.check_config_ready(self.__config_path, "请先初始化配置文件、在数据库创建相应的数据表")
+            else:
+                self.__config_path = tookitsApp.config_file_path
+
+        self.__env = EnvPrepare(tookitsApp.app_name, self.__config_path)
+        self.__mysql_config = self.__env.get_config('mysql_info_online_alarm')
+        self.sql_helper = MysqlHelper(self.__mysql_config)
+
+    def query_sql(self, query_where) -> list:
+        list_api = self.sql_helper.query("SELECT x.* FROM sec_admin.api_sec_output_send x WHERE bot_process in (%s)" % query_where)
+        # logger.debug("Result:[%s]" % json.dumps(list_api))
+        logger.debug("Len: %s" % str(len(list_api)))
+        return list_api
+
+    @staticmethod
+    def __read_all_data(file_path):
+        fp = open(file_path)
+        data_list = fp.readlines()
+        fp.close()
+        ns_list = []
+        for item in data_list:
+            ns_list.append(str(item).strip())
+        return ns_list
+
+    def worker(self, file_path, batch_size=10):
+        data_list = self.__read_all_data(file_path)
+        final_list = []
+        for i in range(0, len(data_list), batch_size):
+            # 获取当前批次的数据
+            batch_data = data_list[i:i + batch_size]
+            sql_where = "'%s'" % "', '".join(batch_data)
+
+            kylin_list = self.query_sql(sql_where)
+            final_list.extend(kylin_list)
+
+        fp = open("%s.csv" % file_path, "w+")
+        fp.write("source,host,name_space")
+        for item in list(set(final_list)):
+            fp.write("%s,%s,%s\r\n" % (item['source'], item['host'], item['name_space']))
+        fp.close()
+
+
+if __name__ == '__main__':
+    try:
+        fire.Fire(DbQueryDemo)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())
+

toolkits-0.2.7/src/toolkits/libs_core/config_groups_helper.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+
+class ConfigGroupsHelper:
+    """
+    ConfigGroupsHelper
+    """
+    def __init__(self, config_groups, config_groups_type="list"):
+        self.__group_config = {}
+        if config_groups_type == "list":
+            self.__init_config_list(config_groups)
+        elif config_groups_type == "dict":
+            self.__init_config_dict(config_groups)
+
+    def __init_config_dict(self, config_groups_dict, talk_groups=None):
+        init_groups = {}
+        chat_groups = dict(config_groups_dict)
+        if talk_groups:
+            if type(talk_groups) is dict:
+                chat_groups = {
+                    "default": talk_groups
+                }
+
+        if chat_groups:
+            for item in chat_groups.keys():
+                init_groups[item] = chat_groups[item]
+
+        self.__group_config = init_groups
+
+    def __init_config_list(self, config_groups, talk_groups=None):
+        init_groups = {}
+        chat_groups = config_groups
+        if talk_groups:
+            if type(talk_groups) is dict:
+                chat_groups = [talk_groups]
+            else:
+                chat_groups = talk_groups
+
+        if chat_groups:
+            for item in chat_groups:
+                init_groups[item['name']] = item
+
+        self.__group_config = init_groups
+
+    def get_config_by_name(self, config_name):
+        config = None
+        if config_name not in self.__group_config.keys():
+            err_msg = "配置名称不存在，请检查！！"
+            print(err_msg)
+            exit(1)
+        else:
+            config = self.__group_config[config_name]
+        return config
+
+    def list_config_groups(self, config_name=""):
+        groups = self.__group_config.keys()
+        print("%s-配置组:\n" % config_name)
+        for item_group in groups:
+            desc = self.__group_config[item_group]['desc']
+            print("\tName: %s\tDesc:[%s]" % (item_group, desc))

toolkits-0.2.7/src/toolkits/libs_core/config_helper.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+from .load_module import LoadModule
+
+
+class ConfigHelpers:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_config(config_file):
+        app = LoadModule()
+        config_info = app.load_from_file(config_file)
+        return config_info
+
+    @staticmethod
+    def get_config_by_name(config_data, config_name):
+        config_info = None
+        if config_data:
+            config_info = config_data.__dict__[config_name]
+
+        return config_info
+

toolkits-0.2.7/src/toolkits/libs_core/env_prepare.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+import os
+import platform
+import sys
+import traceback
+
+import fire
+import jmespath
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory
+from toolkits.system.shell_helper import exec_shell
+from unipath import Path
+
+from .config_helper import ConfigHelpers
+from .mysql_helper import MysqlHelper
+
+logger = log("EnvPrepare")
+
+
+class EnvPrepare:
+    config_file_path = None
+
+    def __init__(self, app_name, config_full_path=None, config_global_name="config_global"):
+        self.__base_path = get_script_directory()
+        self.__config_global_name = config_global_name
+        self.__user_home_path = os.path.expanduser("~/.config")
+        self.__app_name = app_name
+        self.__config_full_path = None
+        self.__config_path = "%s/%s/" % (self.__get_workdir(), self.__app_name)
+        if config_full_path:
+            if Path(config_full_path).exists():
+                self.__config_full_path = config_full_path
+            else:
+                logger.error("Config was not exist:[%s]" % config_full_path)
+
+        self.__tip_message = "Initialized: finished, please change the default's config to " \
+                             "adapt your environment! "
+        self.__default_path_tip = "Default config path: %s" % self.__config_path
+        self.__dict_config = {}
+
+    def __get_workdir(self):
+        app_data = self.__user_home_path
+        if platform.system().lower() == 'windows':  # print("windows")
+            app_data = "%s\\.config" % os.getenv("APPDATA")
+        elif platform.system().lower() == 'linux':  # print("linux")
+            app_data = self.__user_home_path
+        return app_data
+
+    def check_env(self):
+        # check env-config
+        if not Path(self.__config_path).exists():
+            Path(self.__config_path).mkdir(parents=True)
+            src_path = "%s/../config/" % self.__base_path
+            self.init_default_config("directory", src_path, self.__config_path)
+            print("%s\n%s" % (self.__tip_message, self.__default_path_tip))
+            logger.error("%s\n%s" % (self.__tip_message, self.__default_path_tip))
+
+    def check_config_ready(self, config_path, error_message):
+        if not Path(config_path).exists():
+            msg = error_message + "\n" + self.__default_path_tip
+            print(msg)
+            logger.error(msg)
+            # raise Exception(msg)
+            exit(-1)
+
+    def get_config_path(self, config_file_name):
+        return "%s/%s" % (self.__config_path, config_file_name)
+
+    def get_config_from_file(self, config_path: str, config_key: str, force_reload: bool = False) -> dict:
+        if config_path not in self.__dict_config or force_reload is True:
+            config_data = ConfigHelpers.load_config(config_path)
+            self.__dict_config[config_path] = config_data
+        else:
+            config_data = self.__dict_config[config_path]
+
+        return ConfigHelpers.get_config_by_name(config_data, config_key)
+
+    def __get_default_config_file(self):
+        running_app = os.path.split(os.path.realpath(sys.argv[0]))[1]
+        if running_app[-3:] == ".py" or running_app[-4:] == ".pyc":
+            path_working = os.path.split(os.path.realpath(sys.argv[0]))[0]
+        else:
+            path_working = os.getcwd()
+
+        config_path_working = "%s/config.py" % path_working
+        if not Path(config_path_working).exists():
+            config_path_working = "%s/%s/config.py" % (self.__user_home_path, self.__app_name)
+        return config_path_working
+
+    def get_default_config(self, config_key: str, force_reload: bool = False) -> dict:
+        if EnvPrepare.config_file_path:
+            config_path_working = EnvPrepare.config_file_path
+        else:
+            config_path_working = self.__get_default_config_file()
+        logger.debug("config_path_working: %s", config_path_working)
+        config_data = None
+        if Path(config_path_working).exists():
+            config_data = self.get_config_from_file(config_path_working, config_key, force_reload)
+        return config_data
+
+    def get_config(self, config_key: str, force_reload: bool = False):
+        if self.__config_full_path:
+            config_data = self.get_config_from_file(self.__config_full_path, self.__config_global_name, force_reload)
+        else:
+            config_data = self.get_default_config(self.__config_global_name, force_reload)
+        return jmespath.search(config_key, config_data)
+
+    @staticmethod
+    def init_default_config(init_type, src_path, dest_path):
+        init_status = False
+        if init_type == 'file':
+            if platform.system().lower() == 'windows':
+                cmd_copy = "XCOPY /S /-Y %s %s " % (str(src_path).replace("/", "\\"), str(dest_path).replace("/", "\\"))
+            elif platform.system().lower() == 'linux':
+                cmd_copy = "cp -arf %s %s " % (src_path, dest_path)
+        elif init_type == 'directory':
+            if platform.system().lower() == 'windows':
+                cmd_copy = "xcopy /S /-Y %s %s " % (str(src_path).replace("/", "\\"), str(dest_path).replace("/", "\\"))
+            elif platform.system().lower() == 'linux':
+                cmd_copy = "cp -arf %s/* %s/ " % (src_path, dest_path)
+        else:
+            return init_status
+
+        logger.info("CopyCMD: %s" % cmd_copy)
+        ret = exec_shell(cmd_copy)
+        if str(ret['exit_code']) == "0":
+            init_status = True
+        return init_status
+
+    @staticmethod
+    def init_db(database_config, sql_execute):
+        try:
+            mysql_conn = MysqlHelper(database_config)
+            mysql_conn.execute(sql_execute)
+        except Exception as ex:
+            logger.error("Error: %s" % ex)
+            logger.error(traceback.format_exc())
+
+
+if __name__ == '__main__':
+    try:
+        fire.Fire(EnvPrepare)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())

toolkits-0.2.7/src/toolkits/libs_core/load_module.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+import imp
+import os
+import traceback
+import uuid
+
+from log4python.Log4python import log
+
+logger = log("LoadModule")
+
+
+class LoadModule(object):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_from_file(file_path):
+        mod_name, file_ext = os.path.splitext(os.path.split(file_path)[-1])
+        py_mod = None
+        if file_ext.lower() == '.py':
+            py_mod = imp.load_source(mod_name, file_path)
+        elif file_ext.lower() == '.pyc':
+            py_mod = imp.load_compiled(mod_name, file_path)
+
+        return py_mod
+
+    @staticmethod
+    def __write_tmp_py(file_path, content):
+        fp = open(file_path, "w+")
+        fp.write(content)
+        fp.close()
+
+    def load_from_string(self, func_string, module_name=None):
+        modules = None
+        try:
+            str_random = str(uuid.uuid4()).replace("-", "")
+            if module_name:
+                file_path = "/tmp/dynamic_%s_%s.py" % (module_name, str_random)
+            else:
+                file_path = "/tmp/dynamic_%s.py" % str_random
+            self.__write_tmp_py(file_path, func_string)
+            modules = self.load_from_file(file_path)
+        except Exception as ex:
+            logger.debug("Error: %s" % ex)
+            logger.debug(traceback.format_exc())
+        return modules

toolkits-0.2.7/src/toolkits/libs_core/mysql_helper.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+import importlib
+import sys
+import traceback
+try:
+    from urllib import parse
+except Exception as ex:
+    import urllib as parse
+
+from log4python.Log4python import log
+from sqlalchemy import Engine
+from sqlalchemy import create_engine
+from sqlalchemy.sql import text
+
+importlib.reload(sys)
+logger = log("MysqlHelper")
+import pymysql
+
+pymysql.version_info = (1, 4, 13, "final", 0)
+pymysql.install_as_MySQLdb()
+
+
+class MysqlHelper:
+    def __init__(self, database_config=None):
+        self.mysql_config = database_config
+        self.__engine = None
+
+    def get_mysql_client(self, config_user=None) -> Engine:
+        '''
+        mysql_db = 'mysql://root:***@10.89.189.48:8027/log_etl'
+        '''
+        engine = None
+        try:
+            if self.__engine is None:
+                if config_user:
+                    config_init = config_user
+                else:
+                    config_init = self.mysql_config
+                mysql_db = 'mysql://%s:%s@%s:%s/%s?charset=utf8' % (config_init['user_name'],
+                                                                    parse.quote_plus(config_init['password']),
+                                                                    config_init['host'],
+                                                                    config_init['port'],
+                                                                    config_init['db_name']
+                                                                    )
+                engine = create_engine(mysql_db, echo=False, pool_recycle=3600, pool_pre_ping=True)
+                self.__engine = engine
+            else:
+                engine = self.__engine
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)
+        return engine
+
+    def execute(self, sql):
+        try:
+            # 创建数据库引擎
+            engine = self.get_mysql_client()
+            # 定义SQL查询语句
+            execute_stmt = text(sql)
+            with engine.connect() as connection:
+                try:
+                    result = connection.execute(execute_stmt)
+                    connection.commit()
+                except Exception as e:
+                    logger.error(traceback.format_exc())
+                    logger.error('deal_msg error: %s' % e)
+                    connection.rollback()
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)
+
+    def query(self, sql) -> list[dict]:
+        try:
+            # 创建数据库引擎
+            engine = self.get_mysql_client()
+
+            # 定义SQL查询语句
+            select_stmt = text(sql)
+
+            # 使用连接执行查询
+            result_list = []
+            with engine.connect() as connection:
+                result = connection.execute(select_stmt)
+
+                # 将查询结果转换为字典列表
+                # 使用fetchall()获取所有结果行，每行是一个元组
+                # 使用namedtuple来获取列名和数据的对应关系
+                columns = result.keys()
+                result_list = [dict(zip(columns, row)) for row in result]
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)
+        return result_list
+
+    @staticmethod
+    def get_insert_schema(filed_names_list):
+        schema_final = ""
+        val_data = ""
+        for item in filed_names_list:
+            schema_final = schema_final + "%s, " % item.strip()
+            val_data = val_data + ":%s, " % item.strip()
+        schema_final = schema_final.strip(" ,")
+        val_data = val_data.strip(" ,")
+        return schema_final, val_data
+
+    def bulk_insert(self, table_name: str, columns: list, data_list: list, batch_size=20000, columns_on_duplicate=[]):
+        """
+        执行分批批量插入操作的通用函数。
+
+        :param columns_on_duplicate: 唯一键冲突时覆盖
+        :param table_name: 要插入数据的表名
+        :param columns: 表的列名列表
+        :param data_list: 要插入的数据列表，其中每个元素是一个包含列值的元组
+        :param batch_size: 每批插入的数据量
+        """
+        # 创建数据库引擎
+        engine = self.get_mysql_client()
+
+        on_duplicate_sql = ""
+        if columns_on_duplicate:
+            tmp_list = []
+            for item in columns_on_duplicate:
+                tmp_list.append("%s=VALUES(%s)" % (item, item))
+            if tmp_list:
+                on_duplicate_sql = "ON DUPLICATE KEY UPDATE %s" % ", ".join(tmp_list)
+
+        # 准备批量插入的SQL语句模板
+        schema_final, placeholders = self.get_insert_schema(columns)
+        insert_stmt = text(f"INSERT INTO {table_name} ({schema_final}) VALUES ({placeholders}) {on_duplicate_sql}")
+        logger.info("BulkInsert:[%s]" % str(insert_stmt))
+
+        try:
+            # 使用连接执行分批批量插入
+            with engine.connect() as connection:
+                try:
+                    for i in range(0, len(data_list), batch_size):
+                        # 获取当前批次的数据
+                        batch_data = data_list[i:i + batch_size]
+                        # 执行批量插入
+                        connection.execute(insert_stmt, batch_data)
+                        logger.info(f"Batch inserted {len(batch_data)} records.")
+
+                    connection.commit()
+                    logger.debug(f"Successfully inserted all data into {table_name},inserted {len(batch_data)} records.")
+                except Exception as e:
+                    logger.error(traceback.format_exc())
+                    logger.error('deal_msg error: %s' % e)
+                    connection.rollback()
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)

{toolkits-0.2.6 → toolkits-0.2.7/src}/toolkits/system/shell_helper.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import _thread
 import argparse
+import asyncio
 import importlib
 import os
 import pprint
@@ -120,6 +121,38 @@ return-val
     return result
 
 
+async def run_command_async(command, timeout=5):
+    process = None
+    try:
+        process = await asyncio.create_subprocess_shell(
+            command,
+
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
+        ret = {
+            "exit_code": process.returncode,
+            "stdout": stdout,
+            "stderr": stderr
+        }
+        return ret
+    except asyncio.TimeoutError:
+        if process and process.returncode is None:
+            process.terminate()
+        ret = {
+            "exit_code": -1,
+            "stdout": None,
+            "stderr": f"Command timed out after {timeout} seconds."
+        }
+        return ret
+
+
+def exec_shell_async(command, timeout=5):
+    loop = asyncio.get_event_loop()
+    return loop.run_until_complete(run_command_async(command, timeout=timeout))
+
+
 def exec_shell(cmd, timeout=0, bufsize=0, executable=None,
                stdin=None, stdout=None, stderr=None,
                preexec_fn=None, close_fds=False, shell=False,

toolkits-0.2.7/src/toolkits/system/test_shell_helper.py

@@ -0,0 +1,2 @@
+def test_exec_shell_async():
+    assert False

toolkits-0.2.7/src/toolkits/tookits_app.py

@@ -0,0 +1,17 @@
+# -*- coding: utf-8 -*-
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory, add_relative_search_path
+
+logger = log("tookitsApp")
+
+
+class tookitsApp:
+    app_name = 'tookits'
+    config_file_path = None
+
+    def __init__(self):
+        self.__base_path = get_script_directory()
+
+
+if __name__ == '__main__':
+    pass

toolkits-0.2.7/src/toolkits/tookits_cli.py

@@ -0,0 +1,126 @@
+# -*- coding: utf-8 -*-
+import sys
+import traceback
+
+import fire
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory
+
+from libs_core.env_prepare import EnvPrepare
+from tookits_app import tookitsApp
+from libs_core.process_admin import ProcessAdmin
+from db_query_demo import DbQueryDemo
+
+logger = log("tookits")
+
+
+class tookits:
+    def __init__(self, config_path=None):
+        self.__base_path = get_script_directory()
+        if config_path is None:
+            self.__env = EnvPrepare(tookitsApp.app_name)
+            self.__env.check_env()
+            self.__config_path = self.__env.get_config_path("config.py")
+            self.__env.check_config_ready(self.__config_path, "请先初始化配置文件、在数据库创建相应的数据表")
+        else:
+            tookitsApp.config_file_path = config_path
+        self.db_query = DbQueryDemo
+        self.__db_list = self.__env.get_config("db_list")
+        # self.auto_stat_stock = AutoStatStock(self.__db_list['dev_ops'])
+        self.__configChatGroups = ConfigGroupsHelper(self.__env.get_config('db_list'), 'dict')
+        self.__process_admin = ProcessAdmin()
+
+    def show_db_list(self):
+        """
+        列表查看数据库信息
+        :return:
+        """
+        self.__configChatGroups.list_config_groups("RDS-连接")
+
+    def init_config(self, path_target):
+        src_path = "%s/config/" % self.__base_path
+        self.__env.init_default_config("directory", src_path, path_target)
+
+    def init_database(self):
+        ddl_sql_list = self.__get_db_ddl_list("policy_status_monitor")
+        for ddl_sql in ddl_sql_list:
+            database_config = self.__env.get_config(self.__config_path, "database_config")
+            self.__env.init_db(database_config['mysql'], ddl_sql)
+
+    def __schedule_tasks(self):
+        self.__process_admin.loop_check_exit()
+        # self.auto_stat_stock.schedule_start()
+
+    def __recall_tasks(self):
+        self.__process_admin.loop_check_exit()
+        logger.info("DBinfo: %s", json.dumps(self.__db_list['dev_ops'], ensure_ascii=True))
+        # app = StrategyRecall(mysql_conn_info=self.__db_list['dev_ops'])
+        # app.process_buy_sale_data("stock_buy_sale_data", "group_test_01")
+
+    def start_all_tasks(self):
+        tasks_list = [
+            self.__schedule_tasks,
+            self.__recall_tasks
+        ]
+        self.__process_admin.processes_start(tasks_list)
+
+    @staticmethod
+    def __get_db_ddl_list(table_name):
+        ddl_sql = ["""CREATE TABLE `%s` (
+  `task_id` varchar(32) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `task_name` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `task_status` varchar(10) COLLATE utf8mb4_unicode_ci NOT NULL COMMENT '0：关闭，1：启用，8：暂停',
+  `desc` varchar(500) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `task_type` varchar(30) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `task_params` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  `task_crontab` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `action_type` varchar(100) COLLATE utf8mb4_unicode_ci NOT NULL,
+  `action_params` text COLLATE utf8mb4_unicode_ci NOT NULL,
+  `event_id` varchar(32) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `check_condition` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `db_config` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  `extra_data` varchar(1000) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+  PRIMARY KEY (`task_id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci;""" % table_name,
+                   """INSERT INTO """ + table_name + """
+(task_id, task_name, task_status, `desc`, task_type, task_params, task_crontab, action_type, action_params, event_id, check_condition, db_config, extra_data)
+VALUES('00065df8eb99442db2808ee19592000x', 'task_50031_dlp_dchat', '0', '策略产出监控-DLP-Dchat安装状态监控', 'sql', 'select mc_rule as event_id, FROM_UNIXTIME(occur_ts/1000,''%%Y-%%m-%%d'') as event_time, count(*) as event_value, "" as event_detail 
+from online_alarm oa 
+where occur_ts >= unix_timestamp(concat(CURDATE(), " 00:00:00")) * 1000 
+and occur_ts < unix_timestamp(concat(ADDDATE(CURDATE(), 1), " 00:00:00")) * 1000 
+and mc_rule = ''20071''
+group by FROM_UNIXTIME(occur_ts/1000,''%%Y-%%m-%%d''),mc_rule
+ limit 1', '15 20 * * *', 'email', '{
+  "event_id": "50031",
+  "event_action": {
+    "ding_talk": {
+      "group_info": {
+        "name": "long_term_query_talk",
+        "desc": "长期查询监控群",
+        "web_hook": "https://oapi.dingtalk.com/robot/send?access_token=69b8e2bae810c9f392b197072699a8241e0a25552291789128cbe5c805cfd5da"
+      },
+      "group_target": "long_term_query_talk"
+    }
+  }
+}', '50031', 'event_value < 5000', '{
+    "user": "db_admin",
+    "pwd": "gYrmHFNOpkIb/UNM5zCuiDScMdCPwsneOoV38RbEYOs=",
+    "host": "10.83.16.15",
+    "port": 8012,
+    "db_name": "sec_admin"
+}', NULL);
+;"""
+                   ]
+        return ddl_sql
+
+
+def main():
+    try:
+        fire.Fire(tookits)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())
+
+
+if __name__ == '__main__':
+    main()

toolkits-0.2.7/tests/test_tookits.py

@@ -0,0 +1,19 @@
+"""
+Tests for `tookits` module.
+"""
+import pytest
+from toolkits import tookits
+
+
+class TestTookits(object):
+
+    @classmethod
+    def setup_class(cls):
+        pass
+
+    def test_something(self):
+        pass
+
+    @classmethod
+    def teardown_class(cls):
+        pass

toolkits-0.2.6/MANIFEST.in

@@ -1,4 +0,0 @@
-# file GENERATED by distutils, do NOT edit
-setup.py
-toolkits\__init__.py
-include toolkits/phone_address/*.dat

toolkits-0.2.6/PKG-INFO

@@ -1,23 +0,0 @@
-Metadata-Version: 2.1
-Name: toolkits
-Version: 0.2.6
-Summary: toolkits for quickly reference
-Home-page: https://github.com/starwithmoon/toolkits
-Author: li_jia_yue
-Author-email: 59727816@qq.com
-License: BSD
-Keywords: toolkits tools
-Classifier: Development Status :: 3 - Alpha
-Classifier: Framework :: Django
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Natural Language :: English
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.9
-
-# toolkits
-toolkits for quickly reference
-
-

toolkits-0.2.6/README.rst

@@ -1,4 +0,0 @@
-# toolkits
-toolkits for quickly reference
-
-

toolkits-0.2.6/setup.cfg

@@ -1,4 +0,0 @@
-[egg_info]
-tag_build = 
-tag_date = 0
-

toolkits-0.2.6/setup.py

@@ -1,45 +0,0 @@
-import sys
-from distutils.core import setup
-from distutils.command.install import INSTALL_SCHEMES
-
-from setuptools import find_packages
-
-PACKAGE = "toolkits"
-NAME = "toolkits"
-DESCRIPTION = "toolkits for quickly reference"
-AUTHOR = "li_jia_yue"
-AUTHOR_EMAIL = "59727816@qq.com"
-URL = "https://github.com/starwithmoon/toolkits"
-VERSION = __import__(PACKAGE).__version__
-
-setup(
-    name=NAME,
-    version=VERSION,
-    description=DESCRIPTION,
-    long_description=open("README.rst").read(),
-    author=AUTHOR,
-    author_email=AUTHOR_EMAIL,
-    license="BSD",
-    url=URL,
-    packages=find_packages(),
-    package_data={
-                # If any package contains *.txt files, include them:
-                '': ['*.dat']},
-    keywords='toolkits tools',
-    classifiers=[
-        'Development Status :: 3 - Alpha',
-        'Framework :: Django',
-        'Intended Audience :: Developers',
-        'License :: OSI Approved :: BSD License',
-        'Natural Language :: English',
-        "Programming Language :: Python :: 2",
-        'Programming Language :: Python :: 2.6',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.9',
-    ],
-    install_requires=[
-        'schedule',
-        'unicodecsv', 'log4python', 'unipath', 'arrow', 'fire'
-    ],
-    zip_safe=False,
-)

toolkits-0.2.6/toolkits.egg-info/PKG-INFO

@@ -1,23 +0,0 @@
-Metadata-Version: 2.1
-Name: toolkits
-Version: 0.2.6
-Summary: toolkits for quickly reference
-Home-page: https://github.com/starwithmoon/toolkits
-Author: li_jia_yue
-Author-email: 59727816@qq.com
-License: BSD
-Keywords: toolkits tools
-Classifier: Development Status :: 3 - Alpha
-Classifier: Framework :: Django
-Classifier: Intended Audience :: Developers
-Classifier: License :: OSI Approved :: BSD License
-Classifier: Natural Language :: English
-Classifier: Programming Language :: Python :: 2
-Classifier: Programming Language :: Python :: 2.6
-Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.9
-
-# toolkits
-toolkits for quickly reference
-
-

toolkits-0.2.6/toolkits.egg-info/SOURCES.txt

@@ -1,47 +0,0 @@
-MANIFEST.in
-README.rst
-setup.py
-toolkits/__init__.py
-toolkits.egg-info/PKG-INFO
-toolkits.egg-info/SOURCES.txt
-toolkits.egg-info/dependency_links.txt
-toolkits.egg-info/not-zip-safe
-toolkits.egg-info/requires.txt
-toolkits.egg-info/top_level.txt
-toolkits/basic/__init__.py
-toolkits/basic/list_helper.py
-toolkits/databases/__init__.py
-toolkits/databases/database_client_util.py
-toolkits/databases/es_client.py
-toolkits/databases/hive_client.py
-toolkits/databases/hive_cmd.py
-toolkits/databases/hive_helper.py
-toolkits/databases/redis_mgmt.py
-toolkits/databases/sql_helper.py
-toolkits/databases/sqlalchemy_helper.py
-toolkits/databases/status_check.py
-toolkits/network/__init__.py
-toolkits/network/ip_helper.py
-toolkits/network/pdi_helper.py
-toolkits/network/send_mail.py
-toolkits/system/__init__.py
-toolkits/system/aes_cipher.py
-toolkits/system/basic_utils.py
-toolkits/system/collections_helper.py
-toolkits/system/crpyt_helper.py
-toolkits/system/dict2xml.py
-toolkits/system/dict_helper.py
-toolkits/system/excel_helper.py
-toolkits/system/file_helper.py
-toolkits/system/load_module.py
-toolkits/system/priority_tasks.py
-toolkits/system/shell_helper.py
-toolkits/system/str_helper.py
-toolkits/system/time_helper.py
-toolkits/system/win32_env.py
-toolkits/system/process_monitor/__init__.py
-toolkits/system/process_monitor/process_monitor.py
-toolkits/system/tasks_deamon/__init__.py
-toolkits/system/tasks_deamon/tasks_controller.py
-toolkits/system/tasks_deamon/tasks_multiprocessing.py
-toolkits/system/tasks_deamon/tasks_process.py

toolkits-0.2.6/toolkits.egg-info/dependency_links.txt

@@ -1 +0,0 @@
-

toolkits-0.2.6/toolkits.egg-info/not-zip-safe

@@ -1 +0,0 @@
-

toolkits-0.2.6/toolkits.egg-info/requires.txt

@@ -1,6 +0,0 @@
-schedule
-unicodecsv
-log4python
-unipath
-arrow
-fire

toolkits-0.2.6/toolkits.egg-info/top_level.txt

@@ -1 +0,0 @@
-toolkits