toolkits 0.2.7__py3-none-any.whl

toolkits/databases/hive_helper.py

@@ -0,0 +1,220 @@
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import os
+import sys
+
+import time
+import uuid
+from os.path import dirname
+
+import arrow
+from log4python.Log4python import log
+import traceback
+
+from toolkits.databases.hive_cmd import HiveCmd
+from toolkits.system.excel_helper import ExcelHelper
+from toolkits.system.shell_helper import exec_shell_with_pipe
+from unipath import Path
+import importlib
+
+importlib.reload(sys)
+logger = log("HiveHelper")
+
+
+class HiveHelper:
+    def __init__(self, work_path_base=None, yarn_queue="root.db.default", db_name="db_info"):
+        if not work_path_base:
+            self.base_path = self.get_run_dir()
+        else:
+            self.base_path = work_path_base
+        self.log_path = "%s/logs" % self.base_path
+        self.yarn_queue = yarn_queue
+        self.db_name = db_name
+        self.sql_file = "%s/config/driver_query.sql" % self.base_path
+
+    @staticmethod
+    def get_run_dir():
+        script_path = sys.argv[0]
+        if script_path[0] != "/":
+            full_path = "%s/%s" % (os.getcwd(), script_path)
+        else:
+            full_path = script_path
+        return os.path.dirname(full_path)
+
+    def get_table_schema(self, tmp_table_name, target_table_name, table_info, time_range, data_file):
+        '''
+        :param table_info:
+        {"fields_schema" :'', "partition_type": 'day|hour'}
+        :param target_table_name:
+        :param tmp_table_name:
+        :param time_range:
+        :param data_file:
+        :return: dict_sql = {
+            "header": content_header,
+            "insert": sql_insert,
+            "delete": sql_delete_tmp_table
+        }:
+        '''
+        partition_time = time_range['time_begin']
+        year = arrow.get(partition_time).format('YYYY')
+        month = arrow.get(partition_time).format('MM')
+        day = arrow.get(partition_time).format('DD')
+        hour = arrow.get(partition_time).format('HH')
+
+        header = '''set mapred.job.queue.name=%s;\nuse %s;\n''' % (self.yarn_queue, self.db_name)
+        sql_create_tmp_table = '''CREATE TABLE %s(
+%s
+)
+ROW FORMAT DELIMITED
+FIELDS TERMINATED BY '\\t'
+STORED AS TEXTFILE;\n'''
+        sql_delete_tmp_table = 'drop table %s;\n'
+        sql_load_data = '''LOAD DATA LOCAL INPATH '%s' INTO TABLE %s;\n'''
+
+        if table_info['partition_type'] == "hour":
+            sql_insert = '''INSERT INTO TABLE %s PARTITION (year='%s', month='%s', day='%s', hour='%s') select * from %s;\n''' % (target_table_name, year, month, day, hour, tmp_table_name)
+        else:
+            sql_insert = '''INSERT INTO TABLE %s PARTITION (year='%s', month='%s', day='%s') select * from %s;\n''' % (target_table_name, year, month, day, tmp_table_name)
+
+        content_header = "%s\n%s\n%s\n" % (
+            header,
+            sql_create_tmp_table % (tmp_table_name, table_info['fields_schema']),
+            sql_load_data % (data_file, tmp_table_name)
+        )
+
+        dict_sql = {
+            "header": content_header,
+            "insert": sql_insert,
+            "tmp_table_delete": sql_delete_tmp_table % tmp_table_name
+        }
+        return dict_sql
+
+    def get_file_path(self, time_range):
+        '''
+        :param time_range:
+        :return:
+        ret_dict = {
+            "table_name": table_name,
+            "file_sql": hive_sql,
+            "file_err": hive_err,
+            "file_out": hive_out,
+            "file_data": data_file
+        }
+        '''
+        begin = arrow.get(time_range['time_begin']).format('YYYYMMDD_HHmmss')
+        table_create_time = arrow.now().format('YYYYMMDDHH')
+        end = arrow.get(time_range['time_end']).format('YYYYMMDD_HHmmss')
+        month_str = arrow.get(time_range['time_end']).format('YYYYMM')
+
+        random_str = str(uuid.uuid4()).replace("-", "")
+        name_post_fix = "%s_%s_%s" % (begin, end, random_str)
+        month_dir = "%s/%s" % (self.log_path, month_str)
+        if not Path(month_dir).exists():
+            Path(month_dir).mkdir(True)
+        data_file = "%s/data_%s.dat" % (month_dir, name_post_fix)
+        table_name = "tmp_%s_%s" % (table_create_time, random_str)
+        hive_sql = "%s/audit_%s.hql" % (month_dir, name_post_fix)
+        hive_err = "%s/audit_%s.err" % (month_dir, name_post_fix)
+        hive_out = "%s/audit_%s.dat" % (month_dir, name_post_fix)
+
+        ret_dict = {
+            "table_name": table_name,
+            "file_sql": hive_sql,
+            "file_err": hive_err,
+            "file_out": hive_out,
+            "file_data": data_file
+        }
+        return ret_dict
+
+    def hive_execute(self, dict_path, content, values):
+        self.write_file(dict_path['file_sql'], content, "w")
+        self.write_file(dict_path['file_data'], values, "w")
+
+        cmd = "hive -f %s 2> %s > %s " % (dict_path['file_sql'], dict_path['file_err'], dict_path['file_out'])
+        logger.debug("CMD:[%s]" % cmd)
+        ret = exec_shell_with_pipe(cmd)
+        logger.debug("CMD_RET:[%s]" % ";".join(ret))
+
+    def hive_insert(self, table_info, data, time_range, action_type=True, event_table_name="empl_access"):
+        rows_warn = []
+        for item in data:
+            rows_warn.append("\t".join(map(str, item)))
+
+        if rows_warn:
+            values = "\n".join(list(set(rows_warn)))
+            dict_path = self.get_file_path(time_range)
+            # logger.debug("FilePath:[%s]" % json.dumps(dict_path, ensure_ascii=False))
+            dict_sql = self.get_table_schema(dict_path['table_name'], event_table_name, table_info,
+                                             time_range, dict_path['file_data'])
+
+            if action_type:
+                content = "%s\n%s\n%s" % (
+                    dict_sql['header'],
+                    dict_sql['insert'],
+                    dict_sql['tmp_table_delete']
+                )
+            else:
+                content = dict_sql['header']
+                logger.debug("Mode:Debug!! Just write data to Tmp table[%s]\n Delete Tmp Table[drop table %s]" %
+                             (dict_path['table_name'], dict_path['table_name']))
+            self.hive_execute(dict_path, content, values)
+        else:
+            logger.debug("input [rows_warn] is empty!! ")
+
+    @staticmethod
+    def write_file(file_path, content, mode="w"):
+        fp = open(file_path, mode)
+        fp.write(content)
+        fp.close()
+
+    def hive_query(self, sql_query, file_name=""):
+        random_str = str(uuid.uuid4()).replace("-", "")
+        query_data_file = "%s/hive_query/hive_query_%s_%s.log" % (self.base_path, file_name, random_str)
+        dir_path = dirname(query_data_file)
+        if not Path(dir_path).exists():
+            try:
+                Path(dir_path).mkdir(parents=True)
+            except Exception as ex:
+                logger.error("Error: %s" % ex)
+                logger.error(traceback.format_exc())
+
+        hive_cmd = HiveCmd(query_data_file, sql_query, self.db_name, self.yarn_queue)
+        logger.debug("Query:[%s]; ResultFile:[%s]" % (sql_query, query_data_file))
+        hive_cmd.query()
+        while True:
+            if hive_cmd.query_is_finished():
+                time.sleep(1)
+                break
+        return query_data_file
+
+    def query(self, sql_query):
+        file_name_time = arrow.now().format('YYYY-MM-DDTHHmmss')
+        result_file = self.hive_query(sql_query, file_name_time)
+        phone_list = []
+
+        if Path(result_file).exists():
+            excel = ExcelHelper(result_file, column_split="\t")
+            phone_list = excel.data
+        return phone_list
+
+    def worker(self):
+        pass
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("action", type=str, help="specify the action [start]")
+        args = parser.parse_args()
+
+        if args.action == "start":
+            app = HiveHelper()
+            app.worker()
+        else:
+            print("Please Check the parameter!!")
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())
+
+

toolkits/databases/redis_mgmt.py

@@ -0,0 +1,95 @@
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import sys
+import redis
+from log4python.Log4python import log
+import traceback
+import importlib
+
+importlib.reload(sys)
+logger = log("RedisMgmt")
+
+host_ip = '192.168.100.110'
+redis_info = {
+    'password': 'password',
+    'host': host_ip,
+    'port': 6307
+}
+
+
+class RedisMgmt:
+    def __init__(self, redis_db=8):
+        self.redisCli = redis.StrictRedis(host=redis_info['host'], port=redis_info['port'],
+                                          password=redis_info['password'], db=int(redis_db))
+
+    def dump_queue(self, store_type, key_source, key_destination):
+        data_list = []
+        while True:
+            data = self.redisCli.lpop(key_source)
+            if data:
+                data_list.append(data)
+            else:
+                break
+
+        if store_type == "redis":
+            for data_item in data_list:
+                self.redisCli.rpush(key_destination, data_item)
+        else:
+            fp = open(key_destination, "a+")
+            content = "\n".join(data_list)
+            fp.write(content)
+            fp.close()
+
+        logger.debug("Dumps Success data number; [%s]" % str(len(data_list)))
+
+    def load_to_redis(self, store_type, key_source, key_destination):
+        data_list = []
+        if store_type == "redis":
+            while True:
+                data = self.redisCli.lpop(key_source)
+                if data:
+                    data_list.append(data)
+                else:
+                    break
+        else:
+            fp = open(key_source, "r")
+            data = fp.readlines()
+            if data:
+                if isinstance(data, list):
+                    data_list.extend(data)
+                else:
+                    data_list.append(data)
+            fp.close()
+
+        for data_item in data_list:
+            self.redisCli.rpush(key_destination, data_item.strip())
+        logger.debug("Loads Success data number; [%s]; Data:[%s]" % (str(len(data_list)),
+                                                                     json.dumps(data_list, ensure_ascii=False)))
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("redis_db", type=str, help="specify the redis_db")
+        parser.add_argument("store_type", type=str, help="specify the store type [file|redis]")
+        parser.add_argument("action_type", type=str, help="specify the action type [load|dump]")
+        parser.add_argument("source", type=str, help="specify the queue_source or file")
+        parser.add_argument("destination", type=str, help="specify the queue_destination or file")
+        args = parser.parse_args()
+
+        redis_mgmt = RedisMgmt(args.redis_db)
+        if args.store_type not in ("file", "redis"):
+            logger.debug("Store Type Error! not in [file, redis]")
+
+        if args.action_type not in ("dump", "load"):
+            logger.debug("Store Type Error! not in [dump, load]")
+
+        if args.action_type == "dump":
+            redis_mgmt.dump_queue(args.store_type, args.source, args.destination)
+        else:
+            redis_mgmt.load_to_redis(args.store_type, args.source, args.destination)
+
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())

toolkits/databases/sql_helper.py

@@ -0,0 +1,291 @@
+# -*- coding: utf-8 -*-
+import argparse
+import pprint
+import re
+import sys
+
+import sqlparse
+from log4python.Log4python import log
+from sqlparse.sql import Where, Comparison, Parenthesis, Identifier, Token, Function
+import traceback
+
+from sqlparse.tokens import DML
+import importlib
+
+importlib.reload(sys)
+logger = log("SqlHelper")
+
+
+class SqlWhere:
+    def __init__(self, sql_dml):
+        self.sql_dml = sql_dml
+        self.where_tokens = None
+        self.condition_data = {
+            "condition": {
+                "key": "",
+                "op": "",
+                "val": []
+            },
+            "or_list": [],
+            "and_list": []
+        }
+        self.where_conditions = None
+
+    @staticmethod
+    def split_or(where_tokens):
+        or_list = []
+        or_item = []
+        found_flag = False
+
+        for item in where_tokens:
+            if item.is_keyword and item.value.upper() == 'WHERE':
+                continue
+
+            if item.is_keyword and item.value.upper() == 'OR':
+                found_flag = True
+                # or_item_tmp = copy.deepcopy(or_item)
+                or_list.append(or_item)
+                or_item = []
+            else:
+                or_item.append(item)
+
+        if found_flag:
+            or_list.append(or_item)
+
+        return or_list
+
+    @staticmethod
+    def split_and(where_tokens):
+        and_list = []
+        and_item = []
+        found_flag = False
+        between_flag = False
+
+        for item in where_tokens:
+            if item.is_keyword and item.value.upper() == 'WHERE':
+                continue
+
+            if item.is_keyword and item.value.upper() == 'BETWEEN':
+                between_flag = True
+
+            if item.is_keyword and item.value.upper() == 'AND' and not between_flag:
+                found_flag = True
+                and_list.append(and_item)
+                and_item = []
+            else:
+                if item.is_keyword and item.value.upper() == 'AND' and between_flag:
+                    between_flag = False
+                and_item.append(item)
+
+        if found_flag:
+            and_list.append(and_item)
+
+        return and_list
+
+    @staticmethod
+    def get_condition(tokens):
+        key_found_flag = False
+        val = []
+        key = ""
+        op = []
+
+        for token in tokens:
+            if token.is_whitespace:
+                continue
+
+            if token.is_keyword and token.value.upper() == 'WHERE':
+                continue
+
+            if isinstance(token, Comparison):
+                key = token.left.value
+                op = ["eq"]
+                val = token.right.value
+                break
+
+            if token.is_keyword:
+                if key_found_flag:
+                    op.append(token.value)
+            else:
+                if not key_found_flag:
+                    key = token.value
+                    key_found_flag = True
+                else:
+                    if isinstance(token, Parenthesis):
+                        tmp_val = token.value.strip("() ").split(",")
+                        val_strip = [item.strip("\"") for item in tmp_val]
+                        val.extend(val_strip)
+                    else:
+                        val.append(token.value)
+                        # val += token.value
+        condition_tmp = {
+            "key": key,
+            "op": "-".join(op).upper(),
+            "val": val
+        }
+
+        return condition_tmp
+
+    def check_sub_conditions(self, where_tokens):
+        sub_tokens = []
+        for token in where_tokens:
+            if isinstance(token, Parenthesis) and (self.split_or(token.tokens) or self.split_and(token.tokens)):
+                # sub_tokens.append(token.tokens)
+                sub_tokens = token.tokens
+        return sub_tokens
+
+    def parse_where_tokens(self, where_tokens):
+        where_group_tmp = {
+            "condition": None,
+            "sub_condition": None,
+            "or_list": [],
+            "and_list": [],
+        }
+        or_list = self.split_or(where_tokens)
+        if or_list:
+            for or_item in or_list:
+                where_group_tmp['or_list'].append(self.parse_where_tokens(or_item))
+        else:
+            and_list = self.split_and(where_tokens)
+            if and_list:
+                for and_item in and_list:
+                    where_group_tmp['and_list'].append(self.parse_where_tokens(and_item))
+            else:
+                sub_tokens = self.check_sub_conditions(where_tokens)
+                if sub_tokens:
+                    where_group_tmp['sub_condition'] = self.parse_where_tokens(sub_tokens)
+                else:
+                    where_group_tmp['condition'] = self.get_condition(where_tokens)
+
+        return where_group_tmp
+
+    def print_where(self):
+        pprint.pprint(self.where_conditions)
+
+    def get_column_condition(self, where_condition):
+        column_condition = []
+        if where_condition['condition']:
+            column_condition.append(where_condition['condition'])
+
+        if where_condition['sub_condition']:
+            column_condition.extend(self.get_column_condition(where_condition['sub_condition']))
+
+        if where_condition['or_list']:
+            for item in where_condition['or_list']:
+                column_condition.extend(self.get_column_condition(item))
+
+        if where_condition['and_list']:
+            for item in where_condition['and_list']:
+                column_condition.extend(self.get_column_condition(item))
+
+        return column_condition
+
+    def get_where_conditions(self):
+        where_tokens_list = self.get_sql_where()
+
+        column_condition_list = []
+        for where in where_tokens_list:
+            tmp_where_tokens = self.parse_where_tokens(where)
+            tmp_list = self.get_column_condition(tmp_where_tokens)
+            column_condition_list.extend(tmp_list)
+
+        return column_condition_list
+
+    def has_dml(self, obj_check, level):
+        sql_dml = None
+        try:
+            if level == 0:
+                return sql_dml
+            else:
+                level -= 1
+                found_dml = False
+                if not isinstance(obj_check, Token):
+                    for token_item in obj_check.tokens:
+                        if token_item.ttype is DML:
+                            found_dml = True
+                            break
+                if found_dml:
+                    sql_dml = obj_check
+                else:
+                    if not isinstance(obj_check, Token):
+                        for token_item in obj_check.tokens:
+                            ret = self.has_dml(token_item, level)
+                            if ret:
+                                sql_dml = token_item
+                                break
+        except Exception as ex:
+            sql_dml = None
+            logger.debug("Error: %s" % ex)
+            logger.debug(traceback.format_exc())
+        finally:
+            return sql_dml
+
+    def get_where_tokens(self, tokens):
+        tokens_where = []
+        sql_dml = False
+        for token_item in tokens:
+            if token_item.ttype is DML:
+                sql_dml = True
+                break
+
+        if not sql_dml:
+            return tokens_where
+
+        for token_item in tokens:
+            if isinstance(token_item, Where):
+                tokens_where.append(token_item)
+            if isinstance(token_item, Identifier):
+                token_check = self.has_dml(token_item, 2)
+                if token_check:
+                    sql_where = self.get_where_tokens(token_check.tokens)
+                    if sql_where:
+                        tokens_where.extend(sql_where)
+
+        return tokens_where
+
+    def search_columns(self, where_list, columns_list):
+        for where_item in where_list:
+            for token in where_item.tokens:
+                for column in columns_list:
+                    if token.is_group:
+                        pass
+                    elif token.is_keyword:
+                        pass
+                    elif token.is_whitespace:
+                        pass
+                    if isinstance(token, Identifier):
+                        pass
+                    if isinstance(token, Comparison):
+                        pass
+                    if isinstance(token, Function):
+                        pass
+
+    def get_sql_where(self):
+        '''
+        xx = "select * from test01 where id=1 or (id=2 and id=4) and id=5 or id = 6 and id not in (4,5,6)"
+        xx = "select * from test01 where id=1 or (id=2 and id=4)"
+        '''
+        sql_query = str(re.sub("--.*?;", " ", self.sql_dml)).replace(";", " ")
+        sql_list = sqlparse.parse(sql_query)
+        where_tokens_list = []
+
+        for sql in sql_list:
+            where_list = re.findall("where", sql.value)
+            where_num = len(where_list)
+            if where_num == 0:
+                print("NO found where.")
+                break
+            where_tokens = self.get_where_tokens(sql.tokens)
+            where_tokens_list.extend(where_tokens)
+
+        return where_tokens_list
+
+
+if __name__ == '__main__':
+    try:
+        xx = "select * from test01 where id=1 or (id=2 and id=4)"
+        sql_where = SqlWhere(xx)
+        conditions_list = sql_where.get_where_conditions()
+        pprint.pprint(conditions_list)
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())

toolkits/databases/sqlalchemy_helper.py

@@ -0,0 +1,71 @@
+# -*- coding: utf-8 -*-
+import argparse
+import os
+import pprint
+import sys
+from log4python.Log4python import log
+import traceback
+import importlib
+importlib.reload(sys)
+logger = log("SqlAlchemyHelper")
+
+
+class SqlAlchemyHelper:
+    def __init__(self):
+        self.base_path = self.__get_script_dir()
+        self.sql_file = "%s/config/driver_query.sql" % self.base_path
+
+    @staticmethod
+    def __get_run_dir():
+        script_path = sys.argv[0]
+        if script_path[0] != "/":
+            full_path = "%s/%s" % (os.getcwd(), script_path)
+        else:
+            full_path = script_path
+        return os.path.dirname(full_path)
+
+    @staticmethod
+    def __get_script_dir():
+        return os.path.dirname(os.path.realpath(__file__))
+
+    @staticmethod
+    def __row2dict(row):
+        d = {}
+        for column in list(row.keys()):
+            # a.encode('utf-8').strip()
+            content = ""
+            try:
+                content = str(row[column]).encode('utf-8')
+            except Exception as ex:
+                logger.debug("ColumnName:[%s]; ColumnContent:[%s]" % (column, pprint.pformat(row[column])))
+                logger.error("Error: %s" % ex)
+                logger.error(traceback.format_exc())
+            finally:
+                d[column] = content
+        return d
+
+    def rows2list(self, result):
+        rows = []
+        for item in result:
+            rows.append(self.__row2dict(item))
+        return rows
+
+    def worker(self):
+        pass
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("action", type=str, help="specify the action [start]")
+        args = parser.parse_args()
+
+        if args.action == "start":
+            app = SqlAlchemyHelper()
+            app.worker()
+        else:
+            print("Please Check the parameter!!")
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())
+