toolkits 0.2.5__tar.gz → 0.2.7__tar.gz

toolkits-0.2.7/PKG-INFO

@@ -0,0 +1,35 @@
+Metadata-Version: 2.1
+Name: toolkits
+Version: 0.2.7
+Summary: toolkits for quickly reference
+Author-Email: li_jia_yue <59727816@qq.com>
+License: MIT
+Requires-Python: >=3.9
+Requires-Dist: fire>=0.6.0
+Requires-Dist: arrow>=1.3.0
+Requires-Dist: pytest>=8.2.2
+Requires-Dist: icecream>=2.1.3
+Requires-Dist: schedule>=1.2.2
+Requires-Dist: jmespath>=1.0.1
+Requires-Dist: unicodecsv>=0.14.1
+Requires-Dist: log4python>=1.0.11
+Requires-Dist: unipath>=1.1
+Description-Content-Type: text/markdown
+
+# toolkits
+## 迁移PDM项目
+1, Create virtual env 
+pdm venv create
+
+2, Install dependency
+pdm install 
+
+3, basic python libs 
+pdm add arrow toolkits requests dingtalkchatbot exchangelib redis PyYAML sqlalchemy kafka-python-ng urllib3==1.26.15 wheel jmespath log4python PyMySQL pytest crython icecream
+
+## 使用包中的Python脚本 
+### 设置Python环境变量 
+      export PYTHONPATH="/home/pythonDemo/toolkits/src/:$PYTHONPATH"
+### 源码目录中不要有[与包名一致的Python脚本文件] 
+### 直接调用Python脚本 
+      python toolkits/db_query_demo.py --config_path="/home/pythonDemo/toolkits/src/toolkits/config/config_demo.py" worker test_01.txt 

toolkits-0.2.7/README.md

@@ -0,0 +1,17 @@
+# toolkits
+## 迁移PDM项目
+1, Create virtual env 
+pdm venv create
+
+2, Install dependency
+pdm install 
+
+3, basic python libs 
+pdm add arrow toolkits requests dingtalkchatbot exchangelib redis PyYAML sqlalchemy kafka-python-ng urllib3==1.26.15 wheel jmespath log4python PyMySQL pytest crython icecream
+
+## 使用包中的Python脚本 
+### 设置Python环境变量 
+      export PYTHONPATH="/home/pythonDemo/toolkits/src/:$PYTHONPATH"
+### 源码目录中不要有[与包名一致的Python脚本文件] 
+### 直接调用Python脚本 
+      python toolkits/db_query_demo.py --config_path="/home/pythonDemo/toolkits/src/toolkits/config/config_demo.py" worker test_01.txt 

toolkits-0.2.7/pyproject.toml

@@ -0,0 +1,35 @@
+[project]
+name = "toolkits"
+version = "0.2.7"
+description = "toolkits for quickly reference"
+authors = [
+    { name = "li_jia_yue", email = "59727816@qq.com" },
+]
+dependencies = [
+    "fire>=0.6.0",
+    "arrow>=1.3.0",
+    "pytest>=8.2.2",
+    "icecream>=2.1.3",
+    "schedule>=1.2.2",
+    "jmespath>=1.0.1",
+    "unicodecsv>=0.14.1",
+    "log4python>=1.0.11",
+    "unipath>=1.1",
+]
+requires-python = ">=3.9"
+readme = "README.md"
+
+[project.license]
+text = "MIT"
+
+[project.scripts]
+toolkits = "toolkits.toolkits_cli:main"
+
+[build-system]
+requires = [
+    "pdm-backend",
+]
+build-backend = "pdm.backend"
+
+[tool.pdm]
+distribution = true

toolkits-0.2.7/src/toolkits/3des/3des.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+import pyDes
+import base64
+
+
+class TripleDesUtils:
+    des_mode = {"CBC": pyDes.CBC, "ECB": pyDes.ECB}
+    des_pad_mode = {"PAD_PKCS5": pyDes.PAD_PKCS5, "PAD_NORMAL": pyDes.PAD_NORMAL}
+
+    def __init__(self, mode, pad_mode, key, iv, pad=None, trans_base64=False):
+        """
+        :param mode: des 加密模式，目前支持 CBC，ECB
+        :param pad_mode: 目前支持 PAD_PKCS5，PAD_NORMAL
+        :param trans_base64: 加密结果是否以 base64 格式输出
+        :param key: 密钥
+        :param iv: 偏移量
+        :param pad:
+        """
+        self.trans_base64 = trans_base64
+        self.k = pyDes.triple_des(key, TripleDesUtils.des_mode.get(mode), iv, pad, TripleDesUtils.des_pad_mode.get(pad_mode))
+
+    def encryption(self, data):
+        """
+        3des 加密
+        说明: 3DES数据块长度为64位，所以IV长度需要为8个字符（ECB模式不用IV），密钥长度为16或24个字符（8个字符以内则结果与DES相同
+        IV与密钥超过长度则截取，不足则在末尾填充'\0'补足
+        :param data: 待加密数据
+        :return:
+        """
+        _encryption_result = self.k.encrypt(data)
+        if self.trans_base64:
+            _encryption_result = self._base64encode(_encryption_result)
+        return _encryption_result
+
+    def decrypt(self, data):
+        """
+        3des 解密
+        :param data: 待解密数据
+        :return:
+        """
+        if self.trans_base64:
+            data = self._base64decode(data)
+        _decrypt_result = self.k.decrypt(data)
+        return _decrypt_result
+
+    @staticmethod
+    def _base64encode(data):
+        """
+        base 64 encode
+        :param data: encode data
+        :return:
+        """
+        try:
+            _b64encode_result = base64.b64encode(data)
+        except Exception as e:
+            raise Exception("base64 encode error:{e}")
+        return _b64encode_result
+
+    @staticmethod
+    def _base64decode(data):
+        """
+        base 64 decode
+        :param data: decode data
+        :return:
+        """
+        try:
+            _b64decode_result = base64.b64decode(data)
+        except Exception as e:
+            raise Exception("base64 decode error:{e}")
+        return _b64decode_result
+
+
+if __name__ == "__main__":
+    test_data = "12345678a"
+    key_a = "uusafeuusafeuusafeuusafe"
+    key_b = "jiayufeuusafeuusafeuusaf"
+    # [12345678a] 3des Result: 2yjtt0Y/c7xEOa9VGetBVA==
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_a, iv="01234567", trans_base64=True)
+    result_a = DesObj.encryption(test_data)
+    print("加密结果: %s" % result_a)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_b, iv="01234567", trans_base64=True)
+    result_b = DesObj.encryption(result_a)
+    print("加密结果: %s" % result_b)
+
+    # result2 = DesObj.decrypt(result)
+    # print("解密结果: %s" % result2)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_b, iv="01234567", trans_base64=True)
+    result_a = DesObj.encryption(test_data)
+    print("加密结果: %s" % result_a)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_a, iv="01234567", trans_base64=True)
+    result_b = DesObj.encryption(result_a)
+    print("加密结果: %s" % result_b)

{toolkits-0.2.5 → toolkits-0.2.7/src}/toolkits/__init__.py

@@ -1,2 +1,2 @@
 __author__ = 'li_jia_yue'
-__version__ = "0.2.5"
+__version__ = "0.2.6"

toolkits-0.2.7/src/toolkits/config/config_demo.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+config_global = {
+    "db_list": {
+        'dev_ops': {
+            'desc': "Home 数据库",
+            'db_name': "dev_ops",
+            'user_name': 'root',
+            'password': '4tKTiDYr81YM',
+            'host': "192.168.100.155",
+            'port': 6606
+        },
+        'online_alarm':  {
+            'desc': "16.15 数据库",
+            'user_name': 'db_admin',
+            'password': '123QWEas!@#',
+            'host': "10.83.16.15",
+            'port': 8012,
+            'db_name': "sec_admin"
+        }
+    },
+    "kafka_list": {
+        "home": {
+            'host': "192.168.100.155",
+            'port': 9092
+        }
+    },
+    "redis_list": {
+        "home": {
+            'password': 's1hKcWqRj9Se',
+            'host': '192.168.100.155',
+            'port': 9379,
+            'db': 1
+        }
+    },
+    "zookeeper_list": {
+        "home": {
+            "bin_path": "/usr/local/dev/kafka/kafka_2.12-2.3.0",
+            "ip": "192.168.100.155",
+            "port": "2184"
+        }
+    }
+}

toolkits-0.2.7/src/toolkits/db_query_demo.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+import json
+import traceback
+
+import fire
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory
+
+from toolkits.libs_core.env_prepare import EnvPrepare
+from toolkits.libs_core.mysql_helper import MysqlHelper
+
+logger = log("DbQueryDemo")
+
+
+class DbQueryDemo:
+    def __init__(self, config_path=None):
+        self.__base_path = get_script_directory()
+        self.__config_path = config_path
+        if config_path is None:
+            if tookitsApp.config_file_path is None:
+                self.__env = EnvPrepare(tookitsApp.app_name)
+                self.__env.check_env()
+                self.__config_path = self.__env.get_config_path("config.py")
+                self.__env.check_config_ready(self.__config_path, "请先初始化配置文件、在数据库创建相应的数据表")
+            else:
+                self.__config_path = tookitsApp.config_file_path
+
+        self.__env = EnvPrepare(tookitsApp.app_name, self.__config_path)
+        self.__mysql_config = self.__env.get_config('mysql_info_online_alarm')
+        self.sql_helper = MysqlHelper(self.__mysql_config)
+
+    def query_sql(self, query_where) -> list:
+        list_api = self.sql_helper.query("SELECT x.* FROM sec_admin.api_sec_output_send x WHERE bot_process in (%s)" % query_where)
+        # logger.debug("Result:[%s]" % json.dumps(list_api))
+        logger.debug("Len: %s" % str(len(list_api)))
+        return list_api
+
+    @staticmethod
+    def __read_all_data(file_path):
+        fp = open(file_path)
+        data_list = fp.readlines()
+        fp.close()
+        ns_list = []
+        for item in data_list:
+            ns_list.append(str(item).strip())
+        return ns_list
+
+    def worker(self, file_path, batch_size=10):
+        data_list = self.__read_all_data(file_path)
+        final_list = []
+        for i in range(0, len(data_list), batch_size):
+            # 获取当前批次的数据
+            batch_data = data_list[i:i + batch_size]
+            sql_where = "'%s'" % "', '".join(batch_data)
+
+            kylin_list = self.query_sql(sql_where)
+            final_list.extend(kylin_list)
+
+        fp = open("%s.csv" % file_path, "w+")
+        fp.write("source,host,name_space")
+        for item in list(set(final_list)):
+            fp.write("%s,%s,%s\r\n" % (item['source'], item['host'], item['name_space']))
+        fp.close()
+
+
+if __name__ == '__main__':
+    try:
+        fire.Fire(DbQueryDemo)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())
+

toolkits-0.2.7/src/toolkits/libs_core/config_groups_helper.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+
+class ConfigGroupsHelper:
+    """
+    ConfigGroupsHelper
+    """
+    def __init__(self, config_groups, config_groups_type="list"):
+        self.__group_config = {}
+        if config_groups_type == "list":
+            self.__init_config_list(config_groups)
+        elif config_groups_type == "dict":
+            self.__init_config_dict(config_groups)
+
+    def __init_config_dict(self, config_groups_dict, talk_groups=None):
+        init_groups = {}
+        chat_groups = dict(config_groups_dict)
+        if talk_groups:
+            if type(talk_groups) is dict:
+                chat_groups = {
+                    "default": talk_groups
+                }
+
+        if chat_groups:
+            for item in chat_groups.keys():
+                init_groups[item] = chat_groups[item]
+
+        self.__group_config = init_groups
+
+    def __init_config_list(self, config_groups, talk_groups=None):
+        init_groups = {}
+        chat_groups = config_groups
+        if talk_groups:
+            if type(talk_groups) is dict:
+                chat_groups = [talk_groups]
+            else:
+                chat_groups = talk_groups
+
+        if chat_groups:
+            for item in chat_groups:
+                init_groups[item['name']] = item
+
+        self.__group_config = init_groups
+
+    def get_config_by_name(self, config_name):
+        config = None
+        if config_name not in self.__group_config.keys():
+            err_msg = "配置名称不存在，请检查！！"
+            print(err_msg)
+            exit(1)
+        else:
+            config = self.__group_config[config_name]
+        return config
+
+    def list_config_groups(self, config_name=""):
+        groups = self.__group_config.keys()
+        print("%s-配置组:\n" % config_name)
+        for item_group in groups:
+            desc = self.__group_config[item_group]['desc']
+            print("\tName: %s\tDesc:[%s]" % (item_group, desc))

toolkits-0.2.7/src/toolkits/libs_core/config_helper.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+from .load_module import LoadModule
+
+
+class ConfigHelpers:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_config(config_file):
+        app = LoadModule()
+        config_info = app.load_from_file(config_file)
+        return config_info
+
+    @staticmethod
+    def get_config_by_name(config_data, config_name):
+        config_info = None
+        if config_data:
+            config_info = config_data.__dict__[config_name]
+
+        return config_info
+

toolkits-0.2.7/src/toolkits/libs_core/env_prepare.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+import os
+import platform
+import sys
+import traceback
+
+import fire
+import jmespath
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory
+from toolkits.system.shell_helper import exec_shell
+from unipath import Path
+
+from .config_helper import ConfigHelpers
+from .mysql_helper import MysqlHelper
+
+logger = log("EnvPrepare")
+
+
+class EnvPrepare:
+    config_file_path = None
+
+    def __init__(self, app_name, config_full_path=None, config_global_name="config_global"):
+        self.__base_path = get_script_directory()
+        self.__config_global_name = config_global_name
+        self.__user_home_path = os.path.expanduser("~/.config")
+        self.__app_name = app_name
+        self.__config_full_path = None
+        self.__config_path = "%s/%s/" % (self.__get_workdir(), self.__app_name)
+        if config_full_path:
+            if Path(config_full_path).exists():
+                self.__config_full_path = config_full_path
+            else:
+                logger.error("Config was not exist:[%s]" % config_full_path)
+
+        self.__tip_message = "Initialized: finished, please change the default's config to " \
+                             "adapt your environment! "
+        self.__default_path_tip = "Default config path: %s" % self.__config_path
+        self.__dict_config = {}
+
+    def __get_workdir(self):
+        app_data = self.__user_home_path
+        if platform.system().lower() == 'windows':  # print("windows")
+            app_data = "%s\\.config" % os.getenv("APPDATA")
+        elif platform.system().lower() == 'linux':  # print("linux")
+            app_data = self.__user_home_path
+        return app_data
+
+    def check_env(self):
+        # check env-config
+        if not Path(self.__config_path).exists():
+            Path(self.__config_path).mkdir(parents=True)
+            src_path = "%s/../config/" % self.__base_path
+            self.init_default_config("directory", src_path, self.__config_path)
+            print("%s\n%s" % (self.__tip_message, self.__default_path_tip))
+            logger.error("%s\n%s" % (self.__tip_message, self.__default_path_tip))
+
+    def check_config_ready(self, config_path, error_message):
+        if not Path(config_path).exists():
+            msg = error_message + "\n" + self.__default_path_tip
+            print(msg)
+            logger.error(msg)
+            # raise Exception(msg)
+            exit(-1)
+
+    def get_config_path(self, config_file_name):
+        return "%s/%s" % (self.__config_path, config_file_name)
+
+    def get_config_from_file(self, config_path: str, config_key: str, force_reload: bool = False) -> dict:
+        if config_path not in self.__dict_config or force_reload is True:
+            config_data = ConfigHelpers.load_config(config_path)
+            self.__dict_config[config_path] = config_data
+        else:
+            config_data = self.__dict_config[config_path]
+
+        return ConfigHelpers.get_config_by_name(config_data, config_key)
+
+    def __get_default_config_file(self):
+        running_app = os.path.split(os.path.realpath(sys.argv[0]))[1]
+        if running_app[-3:] == ".py" or running_app[-4:] == ".pyc":
+            path_working = os.path.split(os.path.realpath(sys.argv[0]))[0]
+        else:
+            path_working = os.getcwd()
+
+        config_path_working = "%s/config.py" % path_working
+        if not Path(config_path_working).exists():
+            config_path_working = "%s/%s/config.py" % (self.__user_home_path, self.__app_name)
+        return config_path_working
+
+    def get_default_config(self, config_key: str, force_reload: bool = False) -> dict:
+        if EnvPrepare.config_file_path:
+            config_path_working = EnvPrepare.config_file_path
+        else:
+            config_path_working = self.__get_default_config_file()
+        logger.debug("config_path_working: %s", config_path_working)
+        config_data = None
+        if Path(config_path_working).exists():
+            config_data = self.get_config_from_file(config_path_working, config_key, force_reload)
+        return config_data
+
+    def get_config(self, config_key: str, force_reload: bool = False):
+        if self.__config_full_path:
+            config_data = self.get_config_from_file(self.__config_full_path, self.__config_global_name, force_reload)
+        else:
+            config_data = self.get_default_config(self.__config_global_name, force_reload)
+        return jmespath.search(config_key, config_data)
+
+    @staticmethod
+    def init_default_config(init_type, src_path, dest_path):
+        init_status = False
+        if init_type == 'file':
+            if platform.system().lower() == 'windows':
+                cmd_copy = "XCOPY /S /-Y %s %s " % (str(src_path).replace("/", "\\"), str(dest_path).replace("/", "\\"))
+            elif platform.system().lower() == 'linux':
+                cmd_copy = "cp -arf %s %s " % (src_path, dest_path)
+        elif init_type == 'directory':
+            if platform.system().lower() == 'windows':
+                cmd_copy = "xcopy /S /-Y %s %s " % (str(src_path).replace("/", "\\"), str(dest_path).replace("/", "\\"))
+            elif platform.system().lower() == 'linux':
+                cmd_copy = "cp -arf %s/* %s/ " % (src_path, dest_path)
+        else:
+            return init_status
+
+        logger.info("CopyCMD: %s" % cmd_copy)
+        ret = exec_shell(cmd_copy)
+        if str(ret['exit_code']) == "0":
+            init_status = True
+        return init_status
+
+    @staticmethod
+    def init_db(database_config, sql_execute):
+        try:
+            mysql_conn = MysqlHelper(database_config)
+            mysql_conn.execute(sql_execute)
+        except Exception as ex:
+            logger.error("Error: %s" % ex)
+            logger.error(traceback.format_exc())
+
+
+if __name__ == '__main__':
+    try:
+        fire.Fire(EnvPrepare)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())

toolkits-0.2.7/src/toolkits/libs_core/load_module.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+import imp
+import os
+import traceback
+import uuid
+
+from log4python.Log4python import log
+
+logger = log("LoadModule")
+
+
+class LoadModule(object):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_from_file(file_path):
+        mod_name, file_ext = os.path.splitext(os.path.split(file_path)[-1])
+        py_mod = None
+        if file_ext.lower() == '.py':
+            py_mod = imp.load_source(mod_name, file_path)
+        elif file_ext.lower() == '.pyc':
+            py_mod = imp.load_compiled(mod_name, file_path)
+
+        return py_mod
+
+    @staticmethod
+    def __write_tmp_py(file_path, content):
+        fp = open(file_path, "w+")
+        fp.write(content)
+        fp.close()
+
+    def load_from_string(self, func_string, module_name=None):
+        modules = None
+        try:
+            str_random = str(uuid.uuid4()).replace("-", "")
+            if module_name:
+                file_path = "/tmp/dynamic_%s_%s.py" % (module_name, str_random)
+            else:
+                file_path = "/tmp/dynamic_%s.py" % str_random
+            self.__write_tmp_py(file_path, func_string)
+            modules = self.load_from_file(file_path)
+        except Exception as ex:
+            logger.debug("Error: %s" % ex)
+            logger.debug(traceback.format_exc())
+        return modules