toolkits 0.2.7__py3-none-any.whl

toolkits/3des/3des.py

@@ -0,0 +1,93 @@
+# -*- coding: utf-8 -*-
+
+import pyDes
+import base64
+
+
+class TripleDesUtils:
+    des_mode = {"CBC": pyDes.CBC, "ECB": pyDes.ECB}
+    des_pad_mode = {"PAD_PKCS5": pyDes.PAD_PKCS5, "PAD_NORMAL": pyDes.PAD_NORMAL}
+
+    def __init__(self, mode, pad_mode, key, iv, pad=None, trans_base64=False):
+        """
+        :param mode: des 加密模式，目前支持 CBC，ECB
+        :param pad_mode: 目前支持 PAD_PKCS5，PAD_NORMAL
+        :param trans_base64: 加密结果是否以 base64 格式输出
+        :param key: 密钥
+        :param iv: 偏移量
+        :param pad:
+        """
+        self.trans_base64 = trans_base64
+        self.k = pyDes.triple_des(key, TripleDesUtils.des_mode.get(mode), iv, pad, TripleDesUtils.des_pad_mode.get(pad_mode))
+
+    def encryption(self, data):
+        """
+        3des 加密
+        说明: 3DES数据块长度为64位，所以IV长度需要为8个字符（ECB模式不用IV），密钥长度为16或24个字符（8个字符以内则结果与DES相同
+        IV与密钥超过长度则截取，不足则在末尾填充'\0'补足
+        :param data: 待加密数据
+        :return:
+        """
+        _encryption_result = self.k.encrypt(data)
+        if self.trans_base64:
+            _encryption_result = self._base64encode(_encryption_result)
+        return _encryption_result
+
+    def decrypt(self, data):
+        """
+        3des 解密
+        :param data: 待解密数据
+        :return:
+        """
+        if self.trans_base64:
+            data = self._base64decode(data)
+        _decrypt_result = self.k.decrypt(data)
+        return _decrypt_result
+
+    @staticmethod
+    def _base64encode(data):
+        """
+        base 64 encode
+        :param data: encode data
+        :return:
+        """
+        try:
+            _b64encode_result = base64.b64encode(data)
+        except Exception as e:
+            raise Exception("base64 encode error:{e}")
+        return _b64encode_result
+
+    @staticmethod
+    def _base64decode(data):
+        """
+        base 64 decode
+        :param data: decode data
+        :return:
+        """
+        try:
+            _b64decode_result = base64.b64decode(data)
+        except Exception as e:
+            raise Exception("base64 decode error:{e}")
+        return _b64decode_result
+
+
+if __name__ == "__main__":
+    test_data = "12345678a"
+    key_a = "uusafeuusafeuusafeuusafe"
+    key_b = "jiayufeuusafeuusafeuusaf"
+    # [12345678a] 3des Result: 2yjtt0Y/c7xEOa9VGetBVA==
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_a, iv="01234567", trans_base64=True)
+    result_a = DesObj.encryption(test_data)
+    print("加密结果: %s" % result_a)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_b, iv="01234567", trans_base64=True)
+    result_b = DesObj.encryption(result_a)
+    print("加密结果: %s" % result_b)
+
+    # result2 = DesObj.decrypt(result)
+    # print("解密结果: %s" % result2)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_b, iv="01234567", trans_base64=True)
+    result_a = DesObj.encryption(test_data)
+    print("加密结果: %s" % result_a)
+    DesObj = TripleDesUtils(mode="CBC", pad_mode="PAD_PKCS5", key=key_a, iv="01234567", trans_base64=True)
+    result_b = DesObj.encryption(result_a)
+    print("加密结果: %s" % result_b)

toolkits/__init__.py

@@ -0,0 +1,2 @@
+__author__ = 'li_jia_yue'
+__version__ = "0.2.6"

toolkits/basic/list_helper.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*-
+import importlib
+import sys
+from log4python.Log4python import log
+
+importlib.reload(sys)
+logger = log("ListHelper")
+
+
+class ListHelper:
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def list_split_by_length(data_list, length_interval):
+        list_final = []
+        data_len = len(data_list)
+        times = data_len / length_interval
+        for index in range(0, times+1):
+            begin = index * length_interval
+            end = begin + length_interval
+            if end > data_len:
+                end = data_len
+            if len(data_list[begin:end]) > 0:
+                list_final.append(data_list[begin:end])
+        return list_final

toolkits/config/config_demo.py

@@ -0,0 +1,43 @@
+# -*- coding: utf-8 -*-
+
+config_global = {
+    "db_list": {
+        'dev_ops': {
+            'desc': "Home 数据库",
+            'db_name': "dev_ops",
+            'user_name': 'root',
+            'password': '4tKTiDYr81YM',
+            'host': "192.168.100.155",
+            'port': 6606
+        },
+        'online_alarm':  {
+            'desc': "16.15 数据库",
+            'user_name': 'db_admin',
+            'password': '123QWEas!@#',
+            'host': "10.83.16.15",
+            'port': 8012,
+            'db_name': "sec_admin"
+        }
+    },
+    "kafka_list": {
+        "home": {
+            'host': "192.168.100.155",
+            'port': 9092
+        }
+    },
+    "redis_list": {
+        "home": {
+            'password': 's1hKcWqRj9Se',
+            'host': '192.168.100.155',
+            'port': 9379,
+            'db': 1
+        }
+    },
+    "zookeeper_list": {
+        "home": {
+            "bin_path": "/usr/local/dev/kafka/kafka_2.12-2.3.0",
+            "ip": "192.168.100.155",
+            "port": "2184"
+        }
+    }
+}

toolkits/databases/database_client_util.py

@@ -0,0 +1,143 @@
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import sys
+import redis
+from log4python.Log4python import log
+import traceback
+from sqlalchemy import create_engine
+import importlib
+
+path_cur = os.path.dirname(os.path.realpath(__file__))
+path_parent = "%s/../" % path_cur
+sys.path.append(path_parent)
+
+from .hive_helper import HiveHelper
+from .hive_client import HiveClient
+importlib.reload(sys)
+logger = log("DatabaseClientUtil")
+
+
+class DatabaseClientUtil:
+    def __init__(self, database_config):
+        """
+database_config:
+hive_config = {
+    "ip": "192.168.100.36",
+    "port": "1627",
+    "username": "test_01",
+    "password": "",
+    "database": "test_db"
+}
+
+mysql_info_online_alarm = {
+    'user': 'db_user',
+    'pwd': 'password!@#',
+    'host': "192.168.100.36",
+    'port': 3306,
+    'db_name': "db_name"
+}
+
+redis_info = {
+    'password': 'xxxpassword',
+    'host': '192.168.100.36',
+    'port': 9027,
+    'db': 1
+}
+
+database_config = {
+    "hive": hive_config,
+    "mysql": mysql_info_online_alarm,
+    "redis": redis_info,
+}
+    """
+        self.base_path = self.__get_script_dir()
+        self.sql_file = "%s/config/driver_query.sql" % self.base_path
+        self.redis_client = None
+        self.conn_mysql = None
+        self.hive_client = None
+        self.hive_config = None
+        self.mysql_config = None
+        self.redis_config = None
+        self.init_database(database_config)
+
+    def init_database(self, config):
+        if config:
+            self.hive_config = config['hive']
+            self.mysql_config = config['mysql']
+            self.redis_config = config['redis']
+
+    def get_hive_client(self, database=None, config_user=None, hive_mode="jdbc",
+                        work_path_base=None, yarn_queue="root.db.default", db_name='db_info'):
+        if config_user:
+            config_init = config_user
+        else:
+            config_init = self.hive_config
+
+        if hive_mode != "jdbc":
+            hive_client = HiveHelper(work_path_base=work_path_base, yarn_queue=yarn_queue, db_name=db_name)
+        else:
+            config_database_name = config_init['database']
+            if database:
+                config_database_name = database
+            hive_client = HiveClient(config_init['ip'], config_init['username'], config_init['password'],
+                                     config_database_name, config_init['port'])
+        return hive_client
+
+    def get_mysql_client(self, config_user=None):
+        '''
+        mysql_db = 'mysql://root:***@10.89.189.48:8027/log_etl'
+        '''
+        if config_user:
+            config_init = config_user
+        else:
+            config_init = self.mysql_config
+        mysql_db = 'mysql://%s:%s@%s:%s/%s?charset=utf8' % (config_init['user'],
+                                                            config_init['pwd'],
+                                                            config_init['host'],
+                                                            config_init['port'],
+                                                            config_init['db_name']
+                                                            )
+        engine = create_engine(mysql_db, echo=False, pool_recycle=3600, pool_pre_ping=True)
+        return engine
+
+    def get_redis_client(self, db_num=8, redis_config=None):
+        if redis_config:
+            config_init = redis_config
+        else:
+            config_init = self.redis_config
+        redis_client = redis.StrictRedis(host=config_init['host'], port=config_init['port'],
+                                         password=config_init['password'], db=db_num)
+        return redis_client
+
+    @staticmethod
+    def __get_run_dir():
+        script_path = sys.argv[0]
+        if script_path[0] != "/":
+            full_path = "%s/%s" % (os.getcwd(), script_path)
+        else:
+            full_path = script_path
+        return os.path.dirname(full_path)
+
+    @staticmethod
+    def __get_script_dir():
+        return os.path.dirname(os.path.realpath(__file__))
+
+    def worker(self):
+        pass
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("action", type=str, help="specify the action [start]")
+        args = parser.parse_args()
+
+        if args.action == "start":
+            app = DatabaseClientUtil()
+            app.worker()
+        else:
+            print("Please Check the parameter!!")
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())

toolkits/databases/es_client.py

@@ -0,0 +1,88 @@
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import sys
+from log4python.Log4python import log
+from elasticsearch import Elasticsearch
+import traceback
+import importlib
+importlib.reload(sys)
+logger = log("AgssQuery")
+
+
+class EsClient:
+    def __init__(self, es_url):
+        self.es_url = es_url  #
+        self.scroll = "1m"
+        self.es = self.es_connect()
+
+    def es_connect(self):
+        es = Elasticsearch(
+                [
+                    self.es_url
+                ],
+                verify_certs=False,
+                timeout=60*60
+        )
+        return es
+
+    def es_search(self, index_name, body_query):
+        res = self.es.search(index=index_name, body=body_query)
+        return res
+
+    def es_search_scroll(self, index_name, body_query):
+        result = []
+        page = self.es.search(index=index_name, body=body_query, scroll=self.scroll)
+        sid = page['_scroll_id']
+        scroll_size = page['hits']['total']
+        result.append(page)
+
+        # Start scrolling
+        while scroll_size > 0:
+            logger.debug("Scrolling...")
+            page = self.es.scroll(scroll_id=sid, scroll=self.scroll)
+            # Update the scroll ID
+            sid = page['_scroll_id']
+            # Get the number of results that we returned in the last scroll
+            scroll_size = len(page['hits']['hits'])
+            logger.debug("scroll size: " + str(scroll_size))
+            result.append(page)
+        return result
+
+
+class AgssQuery:
+    def __init__(self, config, es_url):
+        self.es_client = EsClient(es_url)
+        self.config_audit = config
+
+    @staticmethod
+    def construct_query(config_sys):
+        sys_name = config_sys['desc']
+        rule = config_sys['rule']
+
+        index = rule['es_index']
+        json_search = rule['search']['json']
+        action = rule['action']
+
+        return sys_name, index, json_search, action
+
+    def main_query(self):
+        (sys_name, es_index_name, json_search, action) = self.construct_query(self.config_audit)
+        # query
+        logger.debug("SYS:[%s]; QueryJSON:%s" % (sys_name, json.dumps(json_search, ensure_ascii=False, default=lambda x: x.__dict__)))
+        response = self.es_client.es_search(es_index_name, json_search)
+        logger.debug("SYS:[%s]; Result:%s" % (sys_name, json.dumps(response, ensure_ascii=False, default=lambda x: x.__dict__)))
+        # process
+        ret_process = action(self.config_audit, response)
+
+        return ret_process
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("logFile", type=str, help="specify the log file's path")
+        args = parser.parse_args()
+        print((args.logFile))
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())

toolkits/databases/hive_client.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# hive util with hive server2
+
+import xlwt as xlwt
+from log4python.Log4python import log
+import sys
+import importlib
+
+logger = log("HiveClient")
+
+default_encoding = 'utf-8'
+if sys.getdefaultencoding() != default_encoding:
+    importlib.reload(sys)
+    sys.setdefaultencoding(default_encoding)
+
+
+class HiveClient:
+    def __init__(self, db_host, user, password, database, port=10000, authMechanism="PLAIN"):
+        """
+        create connection to hive server2
+        """
+        import pyhs2
+        import xlwt
+        config = {"mapreduce.job.queuename": "xx.queue.default"}
+        self.conn = pyhs2.connect(host=db_host,
+                                  port=port,
+                                  authMechanism=authMechanism,
+                                  user=user,
+                                  password=password,
+                                  database=database,
+                                  configuration=config
+                                  )
+
+    def query(self, sql):
+
+        """
+        query
+        """
+        with self.conn.cursor() as cursor:
+            cursor.execute(sql)
+            return cursor.fetch()
+
+    def execute(self, sql):
+        try:
+            with self.conn.cursor() as cursor:
+                cursor.execute(sql)
+        except Exception as ex:
+            print(ex)
+
+    def close(self):
+        """
+        close connection
+        """
+        self.conn.close()
+
+
+def writeXlwt(filename,result):
+    book=xlwt.Workbook() #打开一个工作薄
+    sheet1=book.add_sheet('sheel1')#添加一个sheet页
+    for i in range(len(result)+1):
+        if i ==0:
+            sheet1.row(i).write(0, '日期')
+            sheet1.row(i).write(1, '小时')
+            sheet1.row(i).write(2, '楼层')
+            sheet1.row(i).write(3, '店铺号')
+            sheet1.row(i).write(4, '店铺名称')
+            sheet1.row(i).write(5, '人数')
+        else:
+            for a in range(len(result[i-1])):
+                sheet1.row(i).write(a,result[i-1][a])
+    book.save(filename)

toolkits/databases/hive_cmd.py

@@ -0,0 +1,113 @@
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import subprocess
+import sys
+import time
+from os.path import dirname
+from log4python.Log4python import log
+import traceback
+from unipath import Path
+
+from toolkits.system.shell_helper import exec_shell_with_pipe, file_is_used
+import importlib
+
+path_cur = os.path.dirname(os.path.realpath(__file__))
+path_parent = "%s/../" % path_cur
+sys.path.append(path_parent)
+# from system.shellHelper import exec_shell_with_pipe, file_is_used
+
+from threading import Timer
+import _thread
+importlib.reload(sys)
+logger = log("HiveCmd")
+
+
+class HiveCmd:
+    def __init__(self, file_name, sql, db_name, yarn_queue, work_path=None):
+        self.file_name = str(file_name).replace(":", "")
+        if not work_path:
+            self.work_path = dirname(self.file_name)
+        else:
+            self.work_path = work_path
+        self.sql = sql
+        self.hive_hql_file = "%s.hql" % self.file_name
+        self.hive_err_file = "%s.err" % self.file_name
+        self.hive_info = "use %s;\nset mapred.job.queue.name=%s; \n" % (db_name, yarn_queue)
+        logger.debug("HiveCmd:filename[%s]; sql[%s]" % (self.file_name, self.sql))
+
+        path_to_check = dirname(self.file_name)
+        if not Path(path_to_check).exists():
+            Path(path_to_check).mkdir(True)
+
+    def query(self):
+        hql_content = self.hive_info + "%s"
+        hql_cmd = "hive -f %s"
+        Path(self.hive_hql_file).write_file(hql_content % self.sql)
+        cmd = hql_cmd % self.hive_hql_file
+        return exec_shell(cmd, stdout=self.file_name, stderr=self.hive_err_file, work_path=self.work_path)
+
+    def query_is_finished(self):
+        return file_is_used(self.file_name)
+
+
+def run_with_timeout(timeout, default, f, *args, **kwargs):
+    if not timeout:
+        return f(*args, **kwargs)
+    timeout_timer = Timer(timeout, _thread.interrupt_main)
+    try:
+        timeout_timer.start()
+        result = f(*args, **kwargs)
+        return result
+    except KeyboardInterrupt:
+        return default
+    finally:
+        timeout_timer.cancel()
+
+
+def exec_shell(cmd, timeout=0, work_path="", stdout=None, stderr=None):
+    """exeShellWithPipe("grep 'processor' /proc/cpuinfo | sort -u | wc -l")
+    :param stderr:
+    :param stdout:
+    :param work_path:
+    :param timeout:
+    :param cmd:  exec command
+    """
+    if cmd == "" or cmd is None:
+        return "No Cmd Input"
+
+    fp_out = subprocess.PIPE
+    fp_err = subprocess.PIPE
+    if stdout:
+        fp_out = open(stdout, "w+")
+    if stderr:
+        fp_err = open(stderr, "w+")
+
+    if work_path == "":
+        scan_process = subprocess.Popen(cmd, shell=True, stdout=fp_out, stderr=fp_err)
+    else:
+        scan_process = subprocess.Popen(cmd, shell=True, stdout=fp_out, stderr=fp_err, cwd=work_path)
+
+    return_code = None
+    while True:
+        return_code = scan_process.poll()
+        if return_code is None:
+            time.sleep(1)
+        else:
+            break
+    return return_code
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("fileName", type=str, help="specify the sql output file's path")
+        parser.add_argument("sql", type=str, help="specify the sql to query")
+        args = parser.parse_args()
+
+        hive_cmd = HiveCmd(args.fileName, args.sql, "db_name", "root.db.default")
+        ret_code = hive_cmd.query()
+        print(("RetCode:[%s]" % str(ret_code)))
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())