toolkits 0.2.7__py3-none-any.whl

toolkits/databases/status_check.py

@@ -0,0 +1,162 @@
+# -*- coding: utf-8 -*-
+import json
+import sys
+import fire
+from log4python.Log4python import log
+from toolkits.databases.database_client_util import DatabaseClientUtil
+from toolkits.databases.sqlalchemy_helper import SqlAlchemyHelper
+import importlib
+importlib.reload(sys)
+logger = log("StatusCheck")
+
+
+class StatusCheck:
+    def __init__(self,
+                 database_config,
+                 task_info_table="etl_tasks",
+                 field_task_name="task_name",
+                 field_id_name="task_id",
+                 field_status_name="task_status",
+                 status_on_value="1",
+                 status_off_value="0",
+                 redis_keys_prefix="TasksScheduleCheck_",
+                 redis_db_num=1
+                 ):
+        self.tasks_add_list = []
+        self.tasks_edit_list = []
+        self.tasks_delete_list = []
+        self.__tasks_list_redis = {}
+        self.__tasks_list_db = []
+
+        self.__status_on_value = status_on_value
+        self.__status_off_value = status_off_value
+
+        self.__task_info_table = task_info_table
+        self.__task_name = field_task_name
+        self.__task_id_name = field_id_name
+        self.__tasks_keys_prefix = redis_keys_prefix
+        self.__fetch_tasks_sql = "select * from %s where `%s` = '%s' " % (self.__task_info_table,
+                                                                          field_status_name,
+                                                                          self.__status_on_value
+                                                                          )
+
+        self.__tasks_keys_pattern = "%s*" % self.__tasks_keys_prefix
+        self.__database_init = DatabaseClientUtil(database_config)
+        self.__mysql_conn = self.__database_init.get_mysql_client()
+        self.__redis_cli = self.__database_init.get_redis_client(db_num=redis_db_num)
+        self.__sql_helper = SqlAlchemyHelper()
+
+        self.__check_status()
+
+    def get_tasks_list(self):
+        return self.__get_tasks_list_from_db()
+
+    def __add_redis_task(self, task_info):
+        key_id = "%s%s" % (self.__tasks_keys_prefix, task_info[self.__task_id_name])
+        self.__redis_cli.set(key_id, json.dumps(task_info, ensure_ascii=False))
+
+    def __del_redis_task(self, task_info):
+        key_id = "%s%s" % (self.__tasks_keys_prefix, task_info[self.__task_id_name])
+        self.__redis_cli.delete(key_id)
+
+    def __get_tasks_list_from_redis(self):
+        tasks_dict = {}
+        keys = self.__redis_cli.keys(self.__tasks_keys_pattern)
+
+        keys_final = keys
+        if type(keys) is str:
+            keys_final = [keys]
+
+        if type(keys_final) is list:
+            for key_item in keys_final:
+                task_data = self.__redis_cli.get(key_item)
+                task_info = json.loads(task_data)
+                tasks_dict[task_info[self.__task_id_name]] = task_info
+        else:
+            logger.error("Error:Keys type was wrong [%s]" % str(keys))
+
+        return tasks_dict
+
+    def __get_tasks_list_from_db(self):
+        ret = self.__mysql_conn.execute(self.__fetch_tasks_sql)
+        tasks_list = self.__sql_helper.rows2list(ret)
+        return tasks_list
+
+    def get_task_info_by_task_id(self, task_id=None):
+        if task_id:
+            sql = "%s and %s='%s'" % (self.__fetch_tasks_sql, self.__task_id_name, task_id)
+        else:
+            sql = self.__fetch_tasks_sql
+        logger.debug("SqlQuery:[%s]" % sql)
+        ret = self.__mysql_conn.execute(sql)
+
+        rows_data = []
+        if ret:
+            rows_data = self.__sql_helper.rows2list(ret)
+        task_info = {}
+        if len(rows_data) >= 1:
+            task_info = rows_data[0]
+        return task_info
+
+    def __change_task_status(self, tasks_add_list, tasks_edit_list, tasks_delete_list):
+        for item in tasks_add_list:
+            logger.debug("AddTask:[%s]" % json.dumps(item, ensure_ascii=False))
+            self.__add_redis_task(item)
+
+        for item_edit in tasks_edit_list:
+            self.__add_redis_task(item_edit)
+            logger.debug("EditTask:[%s]" % json.dumps(item_edit, ensure_ascii=False))
+
+        for task_item in tasks_delete_list:
+            self.__del_redis_task(task_item)
+            logger.debug("DeleteTask:[%s]" % json.dumps(task_item, ensure_ascii=False))
+
+    def __check_tasks_status_change(self):
+        tasks_add_list = []
+        tasks_edit_list = []
+        tasks_delete_list = []
+        task_id_list = []
+
+        for item_task in self.__tasks_list_db:
+            task_id = item_task[self.__task_id_name]
+            task_id_list.append(task_id)
+
+            if task_id in self.__tasks_list_redis:
+                tasks_edit_list.append(item_task)
+            else:
+                tasks_add_list.append(item_task)
+
+        deleted_id_list = list(set(self.__tasks_list_redis.keys()) - set(task_id_list))
+        for delete_id in deleted_id_list:
+            tasks_delete_list.append(self.__tasks_list_redis[delete_id])
+
+        return tasks_add_list, tasks_edit_list, tasks_delete_list
+
+    def __check_status(self):
+        self.__tasks_list_redis = self.__get_tasks_list_from_redis()
+        self.__tasks_list_db = self.__get_tasks_list_from_db()
+
+        self.tasks_add_list, self.tasks_edit_list, self.tasks_delete_list = self.__check_tasks_status_change()
+        self.__change_task_status(self.tasks_add_list, self.tasks_edit_list, self.tasks_delete_list)
+
+    def check_tasks_status_change(self, task_field_name):
+        tasks_change_list = []
+        for item_task in self.tasks_edit_list:
+            task_id = item_task[self.__task_id_name]
+
+            if task_field_name in item_task:
+                task_field_val = item_task[task_field_name]
+
+                if task_id in self.__tasks_list_redis and task_field_name in self.__tasks_list_redis[task_id]:
+                    if task_field_val != self.__tasks_list_redis[task_id][task_field_name]:
+                        tasks_change_list.append(item_task)
+                else:
+                    logger.error("Error:task_field_name-[%s]" % task_field_name)
+            else:
+                logger.error("Error: field_name:[%s], task_info [%s]" % (task_field_name, json.dumps(item_task)))
+        return tasks_change_list
+
+
+if __name__ == '__main__':
+    fire.Fire(StatusCheck)
+    exit()

toolkits/db_query_demo.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+import json
+import traceback
+
+import fire
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory
+
+from toolkits.libs_core.env_prepare import EnvPrepare
+from toolkits.libs_core.mysql_helper import MysqlHelper
+
+logger = log("DbQueryDemo")
+
+
+class DbQueryDemo:
+    def __init__(self, config_path=None):
+        self.__base_path = get_script_directory()
+        self.__config_path = config_path
+        if config_path is None:
+            if tookitsApp.config_file_path is None:
+                self.__env = EnvPrepare(tookitsApp.app_name)
+                self.__env.check_env()
+                self.__config_path = self.__env.get_config_path("config.py")
+                self.__env.check_config_ready(self.__config_path, "请先初始化配置文件、在数据库创建相应的数据表")
+            else:
+                self.__config_path = tookitsApp.config_file_path
+
+        self.__env = EnvPrepare(tookitsApp.app_name, self.__config_path)
+        self.__mysql_config = self.__env.get_config('mysql_info_online_alarm')
+        self.sql_helper = MysqlHelper(self.__mysql_config)
+
+    def query_sql(self, query_where) -> list:
+        list_api = self.sql_helper.query("SELECT x.* FROM sec_admin.api_sec_output_send x WHERE bot_process in (%s)" % query_where)
+        # logger.debug("Result:[%s]" % json.dumps(list_api))
+        logger.debug("Len: %s" % str(len(list_api)))
+        return list_api
+
+    @staticmethod
+    def __read_all_data(file_path):
+        fp = open(file_path)
+        data_list = fp.readlines()
+        fp.close()
+        ns_list = []
+        for item in data_list:
+            ns_list.append(str(item).strip())
+        return ns_list
+
+    def worker(self, file_path, batch_size=10):
+        data_list = self.__read_all_data(file_path)
+        final_list = []
+        for i in range(0, len(data_list), batch_size):
+            # 获取当前批次的数据
+            batch_data = data_list[i:i + batch_size]
+            sql_where = "'%s'" % "', '".join(batch_data)
+
+            kylin_list = self.query_sql(sql_where)
+            final_list.extend(kylin_list)
+
+        fp = open("%s.csv" % file_path, "w+")
+        fp.write("source,host,name_space")
+        for item in list(set(final_list)):
+            fp.write("%s,%s,%s\r\n" % (item['source'], item['host'], item['name_space']))
+        fp.close()
+
+
+if __name__ == '__main__':
+    try:
+        fire.Fire(DbQueryDemo)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())
+

toolkits/libs_core/config_groups_helper.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+
+
+class ConfigGroupsHelper:
+    """
+    ConfigGroupsHelper
+    """
+    def __init__(self, config_groups, config_groups_type="list"):
+        self.__group_config = {}
+        if config_groups_type == "list":
+            self.__init_config_list(config_groups)
+        elif config_groups_type == "dict":
+            self.__init_config_dict(config_groups)
+
+    def __init_config_dict(self, config_groups_dict, talk_groups=None):
+        init_groups = {}
+        chat_groups = dict(config_groups_dict)
+        if talk_groups:
+            if type(talk_groups) is dict:
+                chat_groups = {
+                    "default": talk_groups
+                }
+
+        if chat_groups:
+            for item in chat_groups.keys():
+                init_groups[item] = chat_groups[item]
+
+        self.__group_config = init_groups
+
+    def __init_config_list(self, config_groups, talk_groups=None):
+        init_groups = {}
+        chat_groups = config_groups
+        if talk_groups:
+            if type(talk_groups) is dict:
+                chat_groups = [talk_groups]
+            else:
+                chat_groups = talk_groups
+
+        if chat_groups:
+            for item in chat_groups:
+                init_groups[item['name']] = item
+
+        self.__group_config = init_groups
+
+    def get_config_by_name(self, config_name):
+        config = None
+        if config_name not in self.__group_config.keys():
+            err_msg = "配置名称不存在，请检查！！"
+            print(err_msg)
+            exit(1)
+        else:
+            config = self.__group_config[config_name]
+        return config
+
+    def list_config_groups(self, config_name=""):
+        groups = self.__group_config.keys()
+        print("%s-配置组:\n" % config_name)
+        for item_group in groups:
+            desc = self.__group_config[item_group]['desc']
+            print("\tName: %s\tDesc:[%s]" % (item_group, desc))

toolkits/libs_core/config_helper.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+from .load_module import LoadModule
+
+
+class ConfigHelpers:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_config(config_file):
+        app = LoadModule()
+        config_info = app.load_from_file(config_file)
+        return config_info
+
+    @staticmethod
+    def get_config_by_name(config_data, config_name):
+        config_info = None
+        if config_data:
+            config_info = config_data.__dict__[config_name]
+
+        return config_info
+

toolkits/libs_core/env_prepare.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+import os
+import platform
+import sys
+import traceback
+
+import fire
+import jmespath
+from log4python.Log4python import log
+from toolkits.system.basic_utils import get_script_directory
+from toolkits.system.shell_helper import exec_shell
+from unipath import Path
+
+from .config_helper import ConfigHelpers
+from .mysql_helper import MysqlHelper
+
+logger = log("EnvPrepare")
+
+
+class EnvPrepare:
+    config_file_path = None
+
+    def __init__(self, app_name, config_full_path=None, config_global_name="config_global"):
+        self.__base_path = get_script_directory()
+        self.__config_global_name = config_global_name
+        self.__user_home_path = os.path.expanduser("~/.config")
+        self.__app_name = app_name
+        self.__config_full_path = None
+        self.__config_path = "%s/%s/" % (self.__get_workdir(), self.__app_name)
+        if config_full_path:
+            if Path(config_full_path).exists():
+                self.__config_full_path = config_full_path
+            else:
+                logger.error("Config was not exist:[%s]" % config_full_path)
+
+        self.__tip_message = "Initialized: finished, please change the default's config to " \
+                             "adapt your environment! "
+        self.__default_path_tip = "Default config path: %s" % self.__config_path
+        self.__dict_config = {}
+
+    def __get_workdir(self):
+        app_data = self.__user_home_path
+        if platform.system().lower() == 'windows':  # print("windows")
+            app_data = "%s\\.config" % os.getenv("APPDATA")
+        elif platform.system().lower() == 'linux':  # print("linux")
+            app_data = self.__user_home_path
+        return app_data
+
+    def check_env(self):
+        # check env-config
+        if not Path(self.__config_path).exists():
+            Path(self.__config_path).mkdir(parents=True)
+            src_path = "%s/../config/" % self.__base_path
+            self.init_default_config("directory", src_path, self.__config_path)
+            print("%s\n%s" % (self.__tip_message, self.__default_path_tip))
+            logger.error("%s\n%s" % (self.__tip_message, self.__default_path_tip))
+
+    def check_config_ready(self, config_path, error_message):
+        if not Path(config_path).exists():
+            msg = error_message + "\n" + self.__default_path_tip
+            print(msg)
+            logger.error(msg)
+            # raise Exception(msg)
+            exit(-1)
+
+    def get_config_path(self, config_file_name):
+        return "%s/%s" % (self.__config_path, config_file_name)
+
+    def get_config_from_file(self, config_path: str, config_key: str, force_reload: bool = False) -> dict:
+        if config_path not in self.__dict_config or force_reload is True:
+            config_data = ConfigHelpers.load_config(config_path)
+            self.__dict_config[config_path] = config_data
+        else:
+            config_data = self.__dict_config[config_path]
+
+        return ConfigHelpers.get_config_by_name(config_data, config_key)
+
+    def __get_default_config_file(self):
+        running_app = os.path.split(os.path.realpath(sys.argv[0]))[1]
+        if running_app[-3:] == ".py" or running_app[-4:] == ".pyc":
+            path_working = os.path.split(os.path.realpath(sys.argv[0]))[0]
+        else:
+            path_working = os.getcwd()
+
+        config_path_working = "%s/config.py" % path_working
+        if not Path(config_path_working).exists():
+            config_path_working = "%s/%s/config.py" % (self.__user_home_path, self.__app_name)
+        return config_path_working
+
+    def get_default_config(self, config_key: str, force_reload: bool = False) -> dict:
+        if EnvPrepare.config_file_path:
+            config_path_working = EnvPrepare.config_file_path
+        else:
+            config_path_working = self.__get_default_config_file()
+        logger.debug("config_path_working: %s", config_path_working)
+        config_data = None
+        if Path(config_path_working).exists():
+            config_data = self.get_config_from_file(config_path_working, config_key, force_reload)
+        return config_data
+
+    def get_config(self, config_key: str, force_reload: bool = False):
+        if self.__config_full_path:
+            config_data = self.get_config_from_file(self.__config_full_path, self.__config_global_name, force_reload)
+        else:
+            config_data = self.get_default_config(self.__config_global_name, force_reload)
+        return jmespath.search(config_key, config_data)
+
+    @staticmethod
+    def init_default_config(init_type, src_path, dest_path):
+        init_status = False
+        if init_type == 'file':
+            if platform.system().lower() == 'windows':
+                cmd_copy = "XCOPY /S /-Y %s %s " % (str(src_path).replace("/", "\\"), str(dest_path).replace("/", "\\"))
+            elif platform.system().lower() == 'linux':
+                cmd_copy = "cp -arf %s %s " % (src_path, dest_path)
+        elif init_type == 'directory':
+            if platform.system().lower() == 'windows':
+                cmd_copy = "xcopy /S /-Y %s %s " % (str(src_path).replace("/", "\\"), str(dest_path).replace("/", "\\"))
+            elif platform.system().lower() == 'linux':
+                cmd_copy = "cp -arf %s/* %s/ " % (src_path, dest_path)
+        else:
+            return init_status
+
+        logger.info("CopyCMD: %s" % cmd_copy)
+        ret = exec_shell(cmd_copy)
+        if str(ret['exit_code']) == "0":
+            init_status = True
+        return init_status
+
+    @staticmethod
+    def init_db(database_config, sql_execute):
+        try:
+            mysql_conn = MysqlHelper(database_config)
+            mysql_conn.execute(sql_execute)
+        except Exception as ex:
+            logger.error("Error: %s" % ex)
+            logger.error(traceback.format_exc())
+
+
+if __name__ == '__main__':
+    try:
+        fire.Fire(EnvPrepare)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())

toolkits/libs_core/load_module.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+import imp
+import os
+import traceback
+import uuid
+
+from log4python.Log4python import log
+
+logger = log("LoadModule")
+
+
+class LoadModule(object):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_from_file(file_path):
+        mod_name, file_ext = os.path.splitext(os.path.split(file_path)[-1])
+        py_mod = None
+        if file_ext.lower() == '.py':
+            py_mod = imp.load_source(mod_name, file_path)
+        elif file_ext.lower() == '.pyc':
+            py_mod = imp.load_compiled(mod_name, file_path)
+
+        return py_mod
+
+    @staticmethod
+    def __write_tmp_py(file_path, content):
+        fp = open(file_path, "w+")
+        fp.write(content)
+        fp.close()
+
+    def load_from_string(self, func_string, module_name=None):
+        modules = None
+        try:
+            str_random = str(uuid.uuid4()).replace("-", "")
+            if module_name:
+                file_path = "/tmp/dynamic_%s_%s.py" % (module_name, str_random)
+            else:
+                file_path = "/tmp/dynamic_%s.py" % str_random
+            self.__write_tmp_py(file_path, func_string)
+            modules = self.load_from_file(file_path)
+        except Exception as ex:
+            logger.debug("Error: %s" % ex)
+            logger.debug(traceback.format_exc())
+        return modules

toolkits/libs_core/mysql_helper.py

@@ -0,0 +1,151 @@
+# -*- coding: utf-8 -*-
+import importlib
+import sys
+import traceback
+try:
+    from urllib import parse
+except Exception as ex:
+    import urllib as parse
+
+from log4python.Log4python import log
+from sqlalchemy import Engine
+from sqlalchemy import create_engine
+from sqlalchemy.sql import text
+
+importlib.reload(sys)
+logger = log("MysqlHelper")
+import pymysql
+
+pymysql.version_info = (1, 4, 13, "final", 0)
+pymysql.install_as_MySQLdb()
+
+
+class MysqlHelper:
+    def __init__(self, database_config=None):
+        self.mysql_config = database_config
+        self.__engine = None
+
+    def get_mysql_client(self, config_user=None) -> Engine:
+        '''
+        mysql_db = 'mysql://root:***@10.89.189.48:8027/log_etl'
+        '''
+        engine = None
+        try:
+            if self.__engine is None:
+                if config_user:
+                    config_init = config_user
+                else:
+                    config_init = self.mysql_config
+                mysql_db = 'mysql://%s:%s@%s:%s/%s?charset=utf8' % (config_init['user_name'],
+                                                                    parse.quote_plus(config_init['password']),
+                                                                    config_init['host'],
+                                                                    config_init['port'],
+                                                                    config_init['db_name']
+                                                                    )
+                engine = create_engine(mysql_db, echo=False, pool_recycle=3600, pool_pre_ping=True)
+                self.__engine = engine
+            else:
+                engine = self.__engine
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)
+        return engine
+
+    def execute(self, sql):
+        try:
+            # 创建数据库引擎
+            engine = self.get_mysql_client()
+            # 定义SQL查询语句
+            execute_stmt = text(sql)
+            with engine.connect() as connection:
+                try:
+                    result = connection.execute(execute_stmt)
+                    connection.commit()
+                except Exception as e:
+                    logger.error(traceback.format_exc())
+                    logger.error('deal_msg error: %s' % e)
+                    connection.rollback()
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)
+
+    def query(self, sql) -> list[dict]:
+        try:
+            # 创建数据库引擎
+            engine = self.get_mysql_client()
+
+            # 定义SQL查询语句
+            select_stmt = text(sql)
+
+            # 使用连接执行查询
+            result_list = []
+            with engine.connect() as connection:
+                result = connection.execute(select_stmt)
+
+                # 将查询结果转换为字典列表
+                # 使用fetchall()获取所有结果行，每行是一个元组
+                # 使用namedtuple来获取列名和数据的对应关系
+                columns = result.keys()
+                result_list = [dict(zip(columns, row)) for row in result]
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)
+        return result_list
+
+    @staticmethod
+    def get_insert_schema(filed_names_list):
+        schema_final = ""
+        val_data = ""
+        for item in filed_names_list:
+            schema_final = schema_final + "%s, " % item.strip()
+            val_data = val_data + ":%s, " % item.strip()
+        schema_final = schema_final.strip(" ,")
+        val_data = val_data.strip(" ,")
+        return schema_final, val_data
+
+    def bulk_insert(self, table_name: str, columns: list, data_list: list, batch_size=20000, columns_on_duplicate=[]):
+        """
+        执行分批批量插入操作的通用函数。
+
+        :param columns_on_duplicate: 唯一键冲突时覆盖
+        :param table_name: 要插入数据的表名
+        :param columns: 表的列名列表
+        :param data_list: 要插入的数据列表，其中每个元素是一个包含列值的元组
+        :param batch_size: 每批插入的数据量
+        """
+        # 创建数据库引擎
+        engine = self.get_mysql_client()
+
+        on_duplicate_sql = ""
+        if columns_on_duplicate:
+            tmp_list = []
+            for item in columns_on_duplicate:
+                tmp_list.append("%s=VALUES(%s)" % (item, item))
+            if tmp_list:
+                on_duplicate_sql = "ON DUPLICATE KEY UPDATE %s" % ", ".join(tmp_list)
+
+        # 准备批量插入的SQL语句模板
+        schema_final, placeholders = self.get_insert_schema(columns)
+        insert_stmt = text(f"INSERT INTO {table_name} ({schema_final}) VALUES ({placeholders}) {on_duplicate_sql}")
+        logger.info("BulkInsert:[%s]" % str(insert_stmt))
+
+        try:
+            # 使用连接执行分批批量插入
+            with engine.connect() as connection:
+                try:
+                    for i in range(0, len(data_list), batch_size):
+                        # 获取当前批次的数据
+                        batch_data = data_list[i:i + batch_size]
+                        # 执行批量插入
+                        connection.execute(insert_stmt, batch_data)
+                        logger.info(f"Batch inserted {len(batch_data)} records.")
+
+                    connection.commit()
+                    logger.debug(f"Successfully inserted all data into {table_name},inserted {len(batch_data)} records.")
+                except Exception as e:
+                    logger.error(traceback.format_exc())
+                    logger.error('deal_msg error: %s' % e)
+                    connection.rollback()
+        except Exception as ex:
+            logger.error(traceback.format_exc())
+            logger.error('deal_msg error: %s' % ex)