toolkits 0.2.7__py3-none-any.whl

toolkits/network/ip_helper.py

@@ -0,0 +1,32 @@
+import socket
+import struct
+
+import sys
+from log4python.Log4python import log
+import importlib
+importlib.reload(sys)
+logger = log("IpHelper")
+
+
+def map_strip(str):
+    tmp = str.lstrip("0")
+    if tmp == "":
+        tmp = "0"
+    return tmp
+
+
+def ip2long(ipstr):
+    split_str = ipstr.split(".")
+    split_str_map = list(map(map_strip, split_str))
+    ip = ".".join(split_str_map)
+    final_ip = 0
+    try:
+        final_ip = struct.unpack("!I", socket.inet_aton(ip))[0]
+    except:
+        logger.error("ErrIP: [%s]" % ipstr)
+    return final_ip
+
+
+def long2ip(ip):
+    ip = int(ip)
+    return socket.inet_ntoa(struct.pack("!I", ip))

toolkits/network/pdi_helper.py

@@ -0,0 +1,206 @@
+# -*- coding: utf-8 -*-
+import argparse
+import json
+import pprint
+import sys
+from os.path import dirname, basename
+import xmltodict
+from log4python.Log4python import log
+import traceback
+from unipath import Path
+from xml.dom.minidom import parseString
+
+from toolkits.system.dict2xml import dicttoxml
+from toolkits.system.shell_helper import exec_shell_with_pipe
+import importlib
+
+importlib.reload(sys)
+logger = log("PdiHelper")
+
+
+class PdiHelper:
+    def __init__(self, pdi_exec):
+        self.pdi_exec_path = pdi_exec  # "/services/pdi/data-integration/pan.sh"
+
+    # 定义xml转json的函数
+    def xmltojson(self, xmlstr):
+        # parse是的xml解析器
+        xmlparse = xmltodict.parse(xmlstr)
+        # json库dumps()是将dict转化成json格式，loads()是将json转化成dict格式。
+        # dumps()方法的ident=1，格式化json
+        jsonstr = json.dumps(xmlparse, indent=1)
+        return jsonstr
+
+    # json转xml函数
+    def jsontoxml(self, jsonstr):
+        # xmltodict库的unparse()json转xml
+        xmlstr = xmltodict.unparse(jsonstr)
+        return xmlstr
+
+    def load_xml(self, xml_config_file):
+        fp = open(xml_config_file)
+        lines = fp.readlines()
+        fp.close()
+
+        config = ""
+        for item in lines:
+            config += item.strip()
+        return config
+
+    def write_xml(self, new_config_file, config_xml):
+        fp_out = open(new_config_file, "w+")
+        fp_out.write(config_xml)
+        fp_out.close()
+
+    def covert_xml(self, xml_config_file, config_set):
+        str_xml = self.load_xml(xml_config_file)
+
+        config_json = self.xmltojson(str_xml)
+        obj_json = json.loads(config_json)
+        print(obj_json)
+        my_item_func = lambda x: x
+        str_xml = dicttoxml(obj_json, attr_type=False, root=False,
+                            item_func=my_item_func, expand_list_item_name_flag=False)
+
+        dom = parseString(str_xml)
+        str_xml = dom.toprettyxml()
+
+        self.write_xml("%s_new" % xml_config_file, str_xml)
+
+    def covert_config(self, xml_config_file, new_config_file, config_set):
+        '''<?xml version="1.0" encoding="utf-8"?>
+    <student>
+    <course>
+    <name>math</name>
+    <score>90</score>
+    </course><info>
+    <name>name</name>
+    <sex>male</sex>
+    </info>
+    <stid>10213</stid>
+    <title>1</title>
+    <title>2</title>
+    <list>
+    <hop>
+    <a>a</a>
+    </hop>
+    <hop>
+    <a>b</a>
+    </hop>
+    </list>
+    </student>
+        # {u'student': {u'info': {u'name': u'name', u'sex': u'male'}, u'course': {u'score': u'90', u'name': u'math'},
+        #  u'stid': u'10213', u'list': {u'hop': [{u'a': u'a'}, {u'a': u'b'}]}, u'title': [u'1', u'2']}}
+        '''
+        ret = False
+        str_xml = self.load_xml(xml_config_file)
+
+        config_json = self.xmltojson(str_xml)
+        obj_config = json.loads(config_json)
+        if self.config_merge(obj_config, config_set):
+            my_item_func = lambda x: x
+            str_xml = dicttoxml(obj_config, attr_type=False, root=False,
+                                item_func=my_item_func, expand_list_item_name_flag=False)
+            dom = parseString(str_xml)
+            str_xml = dom.toprettyxml()
+            self.write_xml(new_config_file, str_xml.replace("&quot;", "\""))
+            ret = True
+        return ret
+
+    def set_dict(self, dst_dict, key_list, val):
+        if len(key_list) > 0:
+            key = key_list[0]
+            del key_list[0]
+            # dst_dict[key] = set_dict_val(dst_dict[key], key_list, val)
+
+            # logger.debug("Key:[%s]" % key)
+            pos_key = key.find("[")
+            if pos_key >= 0:
+                key_search = key[:pos_key]
+                if type(dst_dict) is list:
+                    key_found = False
+                    for item in dst_dict:
+                        if item[key_search] == key[pos_key+1: -1]:
+                            item = self.set_dict(item, key_list, val)
+                            key_found = True
+                    if not key_found:
+                        logger.debug("Error:key_search[%s] Not Found!!" % key)
+                else:
+                    logger.debug("key_search:[%s]" % key_search)
+                    raise
+            else:
+                if key in dict(dst_dict):
+                    dst_dict[key] = self.set_dict(dst_dict[key], key_list, val)
+                else:
+                    logger.debug("Error:key[%s] not found." % key)
+            return dst_dict
+        else:
+            return val
+
+    def config_merge(self, config_template, config_list):
+        ret = False
+        obj_data = None
+        try:
+            for config_item in config_list:
+                key_list = config_item['key'].split(".")
+                val_set = config_item['val']
+                obj_data = config_template
+                config_template = self.set_dict(config_template, key_list, val_set)
+            ret = True
+        except Exception as ex:
+            pprint.pprint(obj_data)
+            logger.debug("Ojbect_Data:%s" % pprint.pformat(obj_data))
+            logger.debug("Error: %s" % ex)
+            logger.debug(traceback.format_exc())
+        finally:
+            return ret
+
+    def fetch_data_from_es(self, pdi_config_file, url, query, data_output_path):
+        log_path = "%s/log/" % dirname(data_output_path)
+        tmp_config_path = "%s/config_tmp/" % dirname(data_output_path)
+        log_file_name = "%s/output_%s.log" % (log_path, basename(data_output_path))
+        if not Path(log_path).exists():
+            Path(log_path).mkdir(True)
+
+        if not Path(tmp_config_path).exists():
+            Path(tmp_config_path).mkdir(True)
+
+        cmd_ret = False
+        try:
+            # modify the config
+            config_set = [
+                {
+                    'key': "transformation.step.name[url_query].fields.field.name[url].nullif",
+                    'val': url
+                },
+                {
+                    'key': "transformation.step.name[url_query].fields.field.name[req].nullif",
+                    'val': query.replace("\n", "")
+                },
+                {
+                    'key': "transformation.step.name[out_put].file.name",
+                    'val': data_output_path
+                }]
+
+            final_kettle_config = "%s/%s_new" % (tmp_config_path, basename(pdi_config_file))
+            if self.covert_config(pdi_config_file, final_kettle_config, config_set):
+                # run process
+                cmd = "%s -file %s -logfile %s" % (self.pdi_exec_path, final_kettle_config, log_file_name)
+                logger.debug("CMD:[%s]" % cmd)
+                exec_shell_with_pipe(cmd)
+                cmd_ret = True
+        except Exception as ex:
+            logger.debug("Error: %s" % ex)
+            logger.debug(traceback.format_exc())
+        finally:
+            return cmd_ret
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("logFile", type=str, help="specify the log file's path")
+        args = parser.parse_args()
+        print((args.logFile))
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())

toolkits/network/send_mail.py

@@ -0,0 +1,105 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import json
+import sys
+import pprint
+import traceback
+from os.path import basename
+from exchangelib import Configuration, Account, Credentials, DELEGATE, Message, Mailbox, FileAttachment
+from log4python.Log4python import log
+import importlib
+
+logger = log("SendMail")
+importlib.reload(sys)
+
+
+class SendMail:
+    def __init__(self, username, password, ews_url, primary_smtp_address):
+        self.username = username
+        self.password = password
+        self.ews_url = ews_url
+        self.primary_smtp_address = primary_smtp_address
+        self.attach_file_list = []
+        self.to_recipients = []
+        self.cc_recipients = []
+        self.bcc_recipients = []
+        self.subject = None
+        self.body = None
+        self.ews_account = self.connect_mail_server()
+
+    def connect_mail_server(self):
+        credentials = Credentials(username=self.username, password=self.password)
+        config = Configuration(service_endpoint=self.ews_url, credentials=credentials)
+        account = Account(primary_smtp_address=self.primary_smtp_address, config=config,
+                          autodiscover=False, access_type=DELEGATE)
+        return account
+
+    @staticmethod
+    def read_file_content(file_path):
+        fp = open(file_path, "r+")
+        lines = fp.readlines()
+        return "".join(lines)
+
+    @staticmethod
+    def list_update(list_container, new_data):
+        if new_data is not None:
+            if type(new_data).__name__ in ["str", "unicode"]:
+                list_container.append(new_data)
+            else:
+                list_container.extend(new_data)
+        return list_container
+
+    def mail_receiver(self, to_recipients, cc_recipients=None, bcc_recipients=None):
+        self.list_update(self.to_recipients, to_recipients)
+        self.list_update(self.cc_recipients, cc_recipients)
+        self.list_update(self.bcc_recipients, bcc_recipients)
+
+    def mail_assemble(self, subject, body, attach_file_list=None):
+        self.list_update(self.attach_file_list, attach_file_list)
+        logger.debug("attach_files:%s" % json.dumps(self.attach_file_list))
+        self.subject = subject
+        self.body = body
+
+    def mail_send(self, receive_mailer, subject, mail_body, attach_file=None):
+        if self.to_recipients is None:
+            return False
+
+        self.mail_receiver(receive_mailer)
+        if attach_file:
+            self.mail_assemble(subject, mail_body, attach_file)
+        else:
+            self.mail_assemble(subject, mail_body)
+
+        m = Message(
+                account=self.ews_account,
+                subject=self.subject,
+                body=self.body,
+                to_recipients=list(set(self.to_recipients)),
+                cc_recipients=list(set(self.cc_recipients)),
+                bcc_recipients=list(set(self.bcc_recipients))
+        )
+
+        for item in self.attach_file_list:
+            logger.debug("attach_file:%s" % str(item))
+            m.attach(FileAttachment(name=basename(item), content=self.read_file_content(item)))
+        m.send()
+        return True
+
+
+if __name__ == '__main__':
+    try:
+        username = "lijiayue"
+        password = "please change to your mail info"
+        ews_url = "https://mail.qq.com/EWS/Exchange.asmx"
+        receive_mail = "lijiayue@qq.com"
+        primary_smtp_address = "lijiayue@qq.com"
+
+        csv_file = "%s/%s" % ("/var/log/", "访问.csv")
+        csv_detail_file = "/var/log/test.txt"
+
+        mail = SendMail(username, password, ews_url, primary_smtp_address)
+        mail.mail_send(receive_mail, "发现违规访问用户", "\n日期：%s; 人员数量：%s；\n访问，见附件：%s\n详细访问信息，见附件：%s\n\n",
+                       [csv_file, csv_detail_file])
+    except Exception as ex:
+        pprint.pprint("Error: %s" % ex)
+        pprint.pprint(traceback.format_exc())

toolkits/system/aes_cipher.py

@@ -0,0 +1,44 @@
+import base64
+from Crypto.Cipher import AES
+from Crypto import Random
+
+
+BS = 16
+pad = lambda s: s + (BS - len(s) % BS) * chr(BS - len(s) % BS)
+unpad = lambda s: s[:-ord(s[len(s)-1:])]
+
+
+class AESCipher:
+    def __init__(self, key, iv_code=None):
+        self.key = key
+        self.iv_random_flag = False
+
+        if iv_code:
+            self.iv = iv_code
+        else:
+            self.iv_random_flag = True
+            self.iv = Random.new().read(AES.block_size)
+
+    def encrypt(self, raw):
+        raw = pad(raw)
+        cipher = AES.new(self.key, AES.MODE_CBC, self.iv)
+
+        if self.iv_random_flag:
+            ret = base64.b64encode(self.iv + cipher.encrypt(raw))
+        else:
+            ret = base64.b64encode(cipher)
+        return ret
+
+    def decrypt(self, enc):
+        enc = base64.b64decode(enc)
+
+        if self.iv_random_flag:
+            self.iv = enc[:16]
+
+        cipher = AES.new(self.key, AES.MODE_CBC, self.iv)
+
+        if self.iv_random_flag:
+            ret = unpad(cipher.decrypt(enc[16:]))
+        else:
+            ret = unpad(cipher.decrypt(enc))
+        return ret

toolkits/system/basic_utils.py

@@ -0,0 +1,20 @@
+import os
+import sys
+
+
+def add_relative_search_path(relative_path_list):
+    caller_file_path = sys._getframe(1).f_code.co_filename
+    path_base = os.path.dirname(os.path.realpath(caller_file_path))
+
+    for item in relative_path_list:
+        path_final = "%s/%s" % (path_base, item)
+        sys.path.append(path_final)
+
+
+def get_script_directory():
+    caller_file_path = sys._getframe(1).f_code.co_filename
+    return os.path.dirname(os.path.realpath(caller_file_path))
+
+
+def get_working_directory():
+    return os.getcwd()

toolkits/system/collections_helper.py

@@ -0,0 +1,72 @@
+# -*- coding: utf-8 -*-
+from functools import reduce
+from collections import Iterator
+from itertools import chain,groupby,product
+
+
+class CollectionsHelper:
+    def __init__(self, src=None):
+        self.src=src
+
+    def toList(self):
+        return list(self.src)
+
+    def toSet(self):
+        return set(self.src)
+
+    def toTuple(self):
+        return tuple(self.src)
+
+    def getSource(self):
+        return self.src
+
+    def map(self,func):
+        return CollectionsHelper(list(map(func, self.src)))
+
+    def filter(self,predicate):
+        return CollectionsHelper(list(filter(predicate, self.src)))
+
+    def reduce(self,func,identity=None):
+        if identity is None:
+            return reduce(func,self.src)
+        else:
+            return reduce(func,self.src,identity)
+
+    def chain(self):
+        return CollectionsHelper(chain.from_iterable(self.src))
+
+    def groupby(self,func=None):
+        return CollectionsHelper([(it[0], list(it[1])) for it in groupby(self.src, func)])
+
+    def product(self,tag):
+        return CollectionsHelper(product(self.src, tag))
+
+    def all(self,predicate):
+        return all([predicate(it) for it in self.src])
+
+    def any(self,predicate):
+        return any([predicate(it) for it in self.src])
+
+    def first(self,predicate=None):
+        if predicate is None:
+            if isinstance(self.src,Iterator):
+                return next(self.src)
+            return next(iter(self.src))
+        else :
+            return next(list(filter(predicate,self.src)))
+
+    def firstOrNone(self,predicate=None):
+        try:
+            if predicate is None:
+                if isinstance(self.src,Iterator):
+                    return next(self.src)
+                return next(iter(self.src))
+            else :
+                return next(list(filter(predicate,self.src)))
+        except StopIteration:
+            return None
+
+
+if __name__ == '__main__':
+    testgroupdata=[{"id":1,"name":"wwb"},{"id":1,"name":"wxa"},{"id":1,"name":"wxb"},{"id":2,"name":"wxc"},{"id":2,"name":"wxd"}]
+    print((CollectionsHelper(testgroupdata).groupby(lambda it:it['id']).toList()))

toolkits/system/crpyt_helper.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+import argparse
+import sys
+from log4python.Log4python import log
+from simplecrypt import encrypt, decrypt
+import traceback
+from unipath import Path
+import importlib
+
+importlib.reload(sys)
+logger = log("MisUserActionEtl")
+
+
+def encrypt_file(file_name):
+    ciphertext = encrypt('password', plaintext)
+
+
+def decrypt_file(file_name):
+    if not Path(file_name).exists():
+        pass
+    plaintext = decrypt('password', ciphertext)
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("action", type=str, help="action: [encrypt|decrypt]")
+        parser.add_argument("file_path", type=str, help="action: [encrypt|decrypt]")
+        args = parser.parse_args()
+
+        if args.action not in ['encrypt', 'decrypt']:
+            print("action value was wrong! please check.[encrypt|decrypt]")
+
+        if args.action == 'encrypt':
+            encrypt_file(args.file_path)
+        elif args.action == 'decrypt':
+            decrypt_file(args.file_path)
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())