toolkits 0.2.7__py3-none-any.whl

toolkits/system/shell_helper.py

@@ -0,0 +1,263 @@
+# -*- coding: utf-8 -*-
+import _thread
+import argparse
+import asyncio
+import importlib
+import os
+import pprint
+import subprocess
+import sys
+import tempfile
+import time
+import traceback
+import uuid
+from multiprocessing import Process
+from threading import Timer
+
+import arrow
+from log4python.Log4python import log
+from unipath import Path
+
+from toolkits.system.file_helper import read_content
+
+importlib.reload(sys)
+logger = log("shellHelper")
+
+
+def change_the_path_split(file_path):
+    path_split_str_win = "\\"
+    path_split_str_linux = "/"
+    if str(sys.platform).find("win") >= 0:
+        path_final = str(file_path).replace(path_split_str_linux, path_split_str_win)
+    else:
+        path_final = str(file_path).replace(path_split_str_win, path_split_str_linux)
+    return path_final
+
+
+def include_path(target_root_directory, relative_path_list):
+    for item in relative_path_list:
+        item = change_the_path_split(item)
+        path_parent = "%s/%s" % (target_root_directory, item)
+        sys.path.append(path_parent)
+
+
+def get_relative_directory_levels(script_file_path, relative_levels):
+    path_cur = os.path.dirname(os.path.realpath(script_file_path))
+    directory_target = "/.." * relative_levels
+    target_root_path = "%s%s" % (path_cur, directory_target)
+    return change_the_path_split(target_root_path)
+
+
+def run_with_timeout(timeout, default, f, *args, **kwargs):
+    if not timeout:
+        return f(*args, **kwargs)
+    timeout_timer = Timer(timeout, _thread.interrupt_main)
+    try:
+        timeout_timer.start()
+        result = f(*args, **kwargs)
+        return result
+    except KeyboardInterrupt:
+        return default
+    finally:
+        timeout_timer.cancel()
+
+
+def exec_cmd(cmd, work_path):
+    exec_shell_with_pipe(cmd, work_path=work_path)
+
+
+def worker(cmd, work_path):
+    p = Process(target=exec_cmd, args=(cmd, work_path))
+    p.start()
+    os._exit(1)
+
+
+def exec_external_cmd_background(cmd, work_path=""):
+    p = Process(target=worker, args=(cmd, work_path))
+    p.start()
+    p.join()
+
+
+def file_is_used(monitor_file):
+    # fuser or lsof to check file's status
+    cmd = "lsof %s" % monitor_file
+    ret = exec_shell_with_pipe(cmd)
+    if not ret:
+        return True
+    else:
+        return False
+
+
+def exec_shell_with_pipe(cmd, timeout=0, work_path=""):
+    """exeShellWithPipe("grep 'processor' /proc/cpuinfo | sort -u | wc -l")
+
+return-val
+     output-lines-list  # line list ['output_01', 'output_02']
+
+    :param work_path:
+    :param timeout:
+    :param cmd:  exec command
+    """
+    result = []
+    none_num = 0
+    if cmd == "" or cmd is None:
+        return "No Cmd Input"
+    if work_path == "":
+        scan_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    else:
+        scan_process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=work_path)
+    while True:
+        if none_num > 3:
+            break
+        if timeout != 0:
+            ret = run_with_timeout(timeout, None, scan_process.stdout.readline)
+        else:
+            ret = scan_process.stdout.readline()
+        if ret == "" or ret is None:
+            none_num += 1
+        else:
+            result.append(ret.strip())
+            none_num = 0
+    return result
+
+
+async def run_command_async(command, timeout=5):
+    process = None
+    try:
+        process = await asyncio.create_subprocess_shell(
+            command,
+
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE
+        )
+        stdout, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout)
+        ret = {
+            "exit_code": process.returncode,
+            "stdout": stdout,
+            "stderr": stderr
+        }
+        return ret
+    except asyncio.TimeoutError:
+        if process and process.returncode is None:
+            process.terminate()
+        ret = {
+            "exit_code": -1,
+            "stdout": None,
+            "stderr": f"Command timed out after {timeout} seconds."
+        }
+        return ret
+
+
+def exec_shell_async(command, timeout=5):
+    loop = asyncio.get_event_loop()
+    return loop.run_until_complete(run_command_async(command, timeout=timeout))
+
+
+def exec_shell(cmd, timeout=0, bufsize=0, executable=None,
+               stdin=None, stdout=None, stderr=None,
+               preexec_fn=None, close_fds=False, shell=False,
+               cwd=None, env=None, universal_newlines=False,
+               startupinfo=None, creationflags=0):
+    """exec_shell("grep 'processor' /proc/cpuinfo | sort -u | wc -l")
+
+
+return-code::
+
+    ret = {
+        "exit_code": return_code,  # 0 or -1
+        "stdout": stdout_msg,  # line list ['output_01', 'output_02']
+        "stderr": stderr_msg   # line list ['output_01', 'output_02']
+    }
+code-end
+
+    :param preexec_fn:
+    :type stdin: object
+    :param executable:
+    :param cwd:
+    :param timeout:
+    :param cmd:  exec command
+    :return: return a dict to caller
+
+    """
+    if cmd == "" or cmd is None:
+        return "No Cmd Input"
+
+    fp_out = subprocess.PIPE
+    fp_err = subprocess.PIPE
+    temp_dir = tempfile.gettempdir()
+    stdout = os.path.join(temp_dir, "stdout_%s" % str(uuid.uuid4()).replace("-", "").upper())
+    stderr = os.path.join(temp_dir, "stderr_%s" % str(uuid.uuid4()).replace("-", "").upper())
+    if stdout:
+        fp_out = open(stdout, "w+")
+    if stderr:
+        fp_err = open(stderr, "w+")
+
+    scan_process = subprocess.Popen(cmd, shell=True, stdout=fp_out, stderr=fp_err, cwd=cwd, bufsize=bufsize,
+                                    executable=executable, stdin=stdin,
+                                    preexec_fn=preexec_fn, close_fds=close_fds,
+                                    env=env, universal_newlines=universal_newlines,
+                                    startupinfo=startupinfo, creationflags=creationflags)
+
+    return_code = None
+    while True:
+        return_code = scan_process.poll()
+        if return_code is None:
+            time.sleep(1)
+        else:
+            break
+
+    fp_out.close()
+    fp_err.close()
+    stdout_msg = read_content(stdout)
+    stderr_msg = read_content(stderr)
+    Path(stderr).remove()
+    Path(stdout).remove()
+
+    ret = {
+        "exit_code": return_code,
+        "stdout": stdout_msg,
+        "stderr": stderr_msg
+    }
+    return ret
+
+
+def dump_data(data_to_dump, file_prefix_name="py_dump_data", dump_path=None):
+    date_str = arrow.now().format('YYYYMMDD_HHmmss')
+    file_name = "%s_%s_%s.dat" % (file_prefix_name, date_str, str(uuid.uuid4()).replace("-", ""))
+    tmp_dir = tempfile.gettempdir()
+    if dump_path:
+        try:
+            if not Path(tmp_dir).exists():
+                Path(tmp_dir).mkdir(parents=True)
+            tmp_dir = dump_path
+        except Exception as ex:
+            logger.error("Error: %s" % ex)
+            logger.error(traceback.format_exc())
+
+    data_dump_file = os.path.join(tmp_dir, file_name)
+    fp = open(data_dump_file, "w+")
+
+    if type(data_to_dump) is str or type(data_to_dump) is str:
+        fp.write("%s\n" % data_to_dump)
+    elif type(data_to_dump) is list:
+        for item in data_to_dump:
+            str_line = pprint.pformat(item)
+            fp.write("%s\n" % str_line)
+    else:
+        str_line = pprint.pformat(data_to_dump)
+        fp.write("%s\n" % str_line)
+    fp.close()
+
+    return data_dump_file
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("logFile", type=str, help="specify the log file's path")
+        args = parser.parse_args()
+        print((args.logFile))
+        # exec_shell()
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())

toolkits/system/str_helper.py

@@ -0,0 +1,187 @@
+# -*- coding: utf-8 -*-
+import hashlib
+import os
+import pprint
+import re
+import sys
+import uuid
+from log4python.Log4python import log
+import importlib
+importlib.reload(sys)
+logger = log("strHelper")
+import esm
+
+
+class StrHelper:
+    def __init__(self):
+        self.__base_path = self.__get_script_dir()
+        self.__sql_file = "%s/config/driver_query.sql" % self.__base_path
+
+    @staticmethod
+    def __element_count(arr, target):
+        return arr.count(target)
+
+    @staticmethod
+    def is_string(obj):
+        try:
+            obj.lower() + obj.title() + obj + ""
+        except Exception as ex:
+            return False
+        else:
+            return True
+
+    @staticmethod
+    def __reg_find_data(reg_pattern, search_data):
+        data_search = []
+        m = re.finditer(reg_pattern, search_data)
+        try:
+            while True:
+                data_match = m.next().group()
+                if data_match:
+                    data_search.append(data_match)
+        except Exception as ex:
+            pass
+        return data_search
+
+    @staticmethod
+    def multi_search(search_string, search_list):
+        """
+        多模匹配搜索字符串
+
+search_string:
+    search_str = u"哎呀，今天在楼下看到了宝马，我老家倒是有养马的，以前的邻居有个奔驰，不对是保时捷，大爷的，都是马"
+search_words:
+    search_words = [u"宝马", u"马", u"奔驰", u"保时捷"]
+result::
+
+    [((33, 39), u'\\u5b9d\\u9a6c'),
+    (36, 39), u'\\u9a6c'),
+    (63, 66), u'\\u9a6c'),
+    (93, 99), u'\\u5954\\u9a70'),
+    (111, 120), u'\\u4fdd\\u65f6\\u6377'),
+    (141, 144), u'\\u9a6c')]
+
+end-code
+
+        :param search_string:
+        :param search_list:
+        :return:
+        """
+        index = esm.Index()
+        for item in search_list:
+            index.enter(item)
+        index.fix()
+        ret = index.query(search_string)
+        return ret
+
+    def __element_repeat_stat(self, data_array):
+        if not data_array:
+            return []
+
+        uniq_data = list(set(data_array))
+        data_list = []
+        for item in uniq_data:
+            count_num = self.__element_count(data_array, item)
+            val_stat = {
+                'emlement': item,
+                'count': count_num
+            }
+            data_list.append(val_stat)
+        return data_list
+
+    @staticmethod
+    def __get_run_dir():
+        script_path = sys.argv[0]
+        if script_path[0] != "/":
+            full_path = "%s/%s" % (os.getcwd(), script_path)
+        else:
+            full_path = script_path
+        return os.path.dirname(full_path)
+
+    @staticmethod
+    def __get_script_dir():
+        return os.path.dirname(os.path.realpath(__file__))
+
+    @staticmethod
+    def md5_str(str_input):
+        from md5 import md5
+        return md5(str_input).hexdigest().upper()
+
+    @staticmethod
+    def md5_file(file_path):
+        hash_md5 = hashlib.md5()
+        with open(file_path, "rb") as f:
+            for chunk in iter(lambda: f.read(4096), b""):
+                hash_md5.update(chunk)
+        return hash_md5.hexdigest()
+
+    @staticmethod
+    def random_str():
+        return str(uuid.uuid4()).replace("-", "")
+
+    def match_phone_info(self, search_data, match_pattern):
+        """
+match phone info
+
+search_data:
+    test_str = ' 175****3754 158****5552 158****5552 13412337575 15812311265 user_id-1334321093 '
+match_pattern:
+        match_pattern = [
+            {
+                "name": "phone",
+                "reg": r'((13[0-9])|(14[5,7])|(15[0-3,5-9])|(17[0,3,5-8])|(18[0-9])|166|190|192|19[6-9]|(147))\d{8}'
+            },
+            {
+                "name": "phone_mask",
+                "reg": r'[0-9]{3}\*{4}[0-9]{4}'
+            },
+            {
+                "name": "user_id",
+                "reg": r'user_id-([0-9]+)'
+            }]
+
+return-val:
+    {
+        "phone": [
+            {
+                "count": 1,
+                "emlement": "13412337575"
+            },
+            {
+                "count": 1,
+                "emlement": "15812311265"
+            }
+        ],
+        "user_id": [
+            {
+                "count": 1,
+                "emlement": "user_id-1334321093"
+            }
+        ],
+        "phone_mask": [
+            {
+                "count": 2,
+                "emlement": "158****5552"
+            },
+            {
+                "count": 1,
+                "emlement": "175****3754"
+            }
+        ]
+    }
+
+end-code
+
+        :param search_data:
+        :param match_pattern:
+        :return:
+        """
+        match_info = {}
+        for item_pattern in match_pattern:
+            phone_reg = item_pattern['reg']
+            pattern_name = item_pattern['name']
+            data_match = self.__reg_find_data(phone_reg, search_data)
+            data_check = self.__element_repeat_stat(data_match)
+            match_info[pattern_name] = data_check
+
+        return match_info

toolkits/system/tasks_deamon/tasks_controller.py

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+import multiprocessing
+import sys
+import time
+
+from multiprocessing import Pool
+import schedule
+from log4python.Log4python import log
+import traceback
+from toolkits.system.time_helper import get_time_range_interval
+import importlib
+
+importlib.reload(sys)
+logger = log("TasksController")
+
+
+class TasksController:
+    def __init__(self, process_num=10):
+        self.process_num = process_num
+        self.pool_process = None
+
+    @staticmethod
+    def run_schedule(exec_worker, exec_time="3:00"):
+        """
+        # schedule.every().minutes.do(self_phone_task)
+        # schedule.every().minutes.do(quick_region_monitor_week_task)
+        # schedule.every().monday.at("5:00").do(quick_region_monitor_week_task)
+        # schedule.every().friday.at("17:32").do(self_phone_task)
+        :return:
+        """
+        schedule.every().day.at(exec_time).do(exec_worker)
+        while True:
+            schedule.run_pending()
+            time.sleep(1)
+
+    @staticmethod
+    def func(msg):
+        print(multiprocessing.current_process().name + '-' + msg)
+
+    def run_redis_tasks(self, exec_worker):
+        self.pool_process = Pool(self.process_num)
+        results = []
+        for range_item in range(0, self.process_num):
+            result = self.pool_process.apply_async(exec_worker, (range_item,), callback=None)
+            results.append(result)
+
+        # 关闭进程池，表示不能再往进程池中添加进程，需要在join之前调用
+        self.pool_process.close()
+        self.pool_process.join()
+
+    def run_time_range(self, exec_worker, time_begin, time_end, time_interval="1d"):
+        self.pool_process = Pool(self.process_num)
+        time_range_list = get_time_range_interval(time_begin, time_end, time_interval)
+
+        results = []
+        for range_item in time_range_list:
+            result = self.pool_process.apply_async(exec_worker, (range_item,), callback=None)
+            results.append(result)
+
+        # 关闭进程池，表示不能再往进程池中添加进程，需要在join之前调用
+        self.pool_process.close()
+        self.pool_process.join()
+
+
+if __name__ == '__main__':
+    try:
+        app = TasksController()
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())

toolkits/system/tasks_deamon/tasks_multiprocessing.py

@@ -0,0 +1,134 @@
+# -*- coding: utf-8 -*-
+import os
+import pprint
+import sys
+import fire
+from log4python.Log4python import log
+import traceback
+import importlib
+
+path_cur = os.path.dirname(os.path.realpath(__file__))
+path_parent = "%s/../../" % path_cur
+sys.path.append(path_parent)
+
+from system.priority_tasks import PriorityTasks
+from databases.database_client_util import DatabaseClientUtil
+from .tasks_controller import TasksController
+importlib.reload(sys)
+logger = log("TasksMultiprocessing")
+
+
+queue_name = None
+queue_switch = None
+func_task = None
+redis_info = None
+
+
+def exec_redis_tasks(task_item):
+    if not queue_name or not queue_switch or not redis_info or not func_task:
+        logger.error("Err: parameters is Wrong")
+        return None
+
+    try:
+        task = PriorityTasks(redis_info, queue_name, work_status_queue_name=queue_switch)
+        task.worker(func_task)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())
+
+
+class TasksMultiprocessing:
+    def __init__(self, tasks_queue_name, tasks_switch, process_num=10):
+        global queue_name, queue_switch
+        queue_name = tasks_queue_name
+        queue_switch = tasks_switch
+        self.__base_path = self.__get_script_dir()
+        self.__database_init = None
+        self.__redis_cli = None
+        self.__db_info = None
+        self.__process_num = process_num
+
+    def init_db(self, database_info, process_num=10):
+        """
+        :param database_info:
+        :param process_num:
+        :return:
+        """
+        self.__db_info = database_info
+        self.__database_init = DatabaseClientUtil(database_info)
+        self.__redis_cli = self.__database_init.get_redis_client(db_num=database_info['redis']['db'])
+        self.__process_num = process_num
+        self.start_tasks()
+
+    @staticmethod
+    def __get_run_dir():
+        script_path = sys.argv[0]
+        if script_path[0] != "/":
+            full_path = "%s/%s" % (os.getcwd(), script_path)
+        else:
+            full_path = script_path
+        return os.path.dirname(full_path)
+
+    @staticmethod
+    def __get_script_dir():
+        return os.path.dirname(os.path.realpath(__file__))
+
+    def get_tasks_queue_info(self):
+        global queue_name, queue_switch
+        queue_info = {
+            "tasks_redis": {
+                "host": self.__db_info['redis']['host'],
+                "port": self.__db_info['redis']['port'],
+                "db_num": self.__db_info['redis']['db']
+            },
+            "tasks_queue": ["%s_high" % queue_name,
+                            "%s_mid" % queue_name,
+                            "%s_low" % queue_name],
+            "tasks_switch": queue_switch
+        }
+        pprint.pprint(queue_info)
+
+    def start_tasks(self):
+        global queue_switch
+        self.__redis_cli.set(queue_switch, "on")
+        logger.debug("Set switch ON [%s]" % queue_switch)
+
+    def __wait_process_stop(self):
+        pass
+
+    def __kill_process(self):
+        pass
+
+    def set_worker(self, func_process):
+        global func_task, redis_info
+        func_task = func_process
+        redis_info = self.__db_info['redis']
+
+    def stop_tasks(self):
+        global queue_switch
+        self.__redis_cli.set(queue_switch, "off")
+        logger.debug("Set switch OFF [%s]" % queue_switch)
+        self.__wait_process_stop()
+        self.__kill_process()
+
+    def daemon_start(self, action):
+        """
+        action = start
+        :param action:
+        :return:
+        """
+        if action == "start":
+            task_controller = TasksController(self.__process_num)
+            task_controller.run_redis_tasks(exec_redis_tasks)
+        else:
+            print("Please Check the parameter!!")
+
+
+if __name__ == '__main__':
+    try:
+        # import pydevd
+        # pydevd.settrace('127.0.0.1', port=6868, stdoutToServer=True, stderrToServer=True)
+        fire.Fire(TasksMultiprocessing)
+    except Exception as ex:
+        logger.error("Error: %s" % ex)
+        logger.error(traceback.format_exc())