toolkits 0.2.7__py3-none-any.whl

toolkits/system/dict2xml.py

@@ -0,0 +1,416 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+"""
+Converts a Python dictionary or other native data type into a valid XML string.
+
+Supports item (`int`, `float`, `long`, `decimal.Decimal`, `bool`, `str`, `unicode`, `datetime`, `none` and other number-like objects) and collection (`list`, `set`, `tuple` and `dict`, as well as iterable and dict-like objects) data types, with arbitrary nesting for the collections. Items with a `datetime` type are converted to ISO format strings. Items with a `None` type become empty XML elements.
+
+This module works with both Python 2 and 3.
+"""
+
+
+
+__version__ = '1.7.4'
+version = __version__
+
+from random import randint
+import collections
+import numbers
+import logging
+from xml.dom.minidom import parseString
+
+
+LOG = logging.getLogger("dicttoxml")
+expand_list_item_name = True
+
+# python 3 doesn't have a unicode type
+try:
+    str
+except:
+    str = str
+
+# python 3 doesn't have a long type
+try:
+    int
+except:
+    long = int
+
+
+def set_debug(debug=True, filename='dicttoxml.log'):
+    if debug:
+        import datetime
+        print(('Debug mode is on. Events are logged at: %s' % (filename)))
+        logging.basicConfig(filename=filename, level=logging.INFO)
+        LOG.info('\nLogging session starts: %s' % (
+            str(datetime.datetime.today()))
+                 )
+    else:
+        logging.basicConfig(level=logging.WARNING)
+        print('Debug mode is off.')
+
+
+def unicode_me(something):
+    """Converts strings with non-ASCII characters to unicode for LOG.
+    Python 3 doesn't have a `unicode()` function, so `unicode()` is an alias
+    for `str()`, but `str()` doesn't take a second argument, hence this kludge.
+    """
+    try:
+        return str(something, 'utf-8')
+    except:
+        return str(something)
+
+
+ids = [] # initialize list of unique ids
+
+def make_id(element, start=100000, end=999999):
+    """Returns a random integer"""
+    return '%s_%s' % (element, randint(start, end))
+
+
+def get_unique_id(element):
+    """Returns a unique id for a given element"""
+    this_id = make_id(element)
+    dup = True
+    while dup:
+        if this_id not in ids:
+            dup = False
+            ids.append(this_id)
+        else:
+            this_id = make_id(element)
+    return ids[-1]
+
+
+def get_xml_type(val):
+    """Returns the data type for the xml type attribute"""
+    if type(val).__name__ in ('str', 'unicode'):
+        return 'str'
+    if type(val).__name__ in ('int', 'long'):
+        return 'int'
+    if type(val).__name__ == 'float':
+        return 'float'
+    if type(val).__name__ == 'bool':
+        return 'bool'
+    if isinstance(val, numbers.Number):
+        return 'number'
+    if type(val).__name__ == 'NoneType':
+        return 'null'
+    if isinstance(val, dict):
+        return 'dict'
+    if isinstance(val, collections.Iterable):
+        return 'list'
+    return type(val).__name__
+
+
+def escape_xml(s):
+    if type(s) in (str, str):
+        s = unicode_me(s) # avoid UnicodeDecodeError
+        s = s.replace('&', '&')
+        s = s.replace('"', '"')
+        s = s.replace('\'', ''')
+        s = s.replace('<', '&lt;')
+        s = s.replace('>', '&gt;')
+    return s
+
+
+def make_attrstring(attr):
+    """Returns an attribute string in the form key="val" """
+    attrstring = ' '.join(['%s="%s"' % (k, v) for k, v in list(attr.items())])
+    return '%s%s' % (' ' if attrstring != '' else '', attrstring)
+
+
+def key_is_valid_xml(key):
+    """Checks that a key is a valid XML name"""
+    LOG.info('Inside key_is_valid_xml(). Testing "%s"' % (unicode_me(key)))
+    test_xml = '<?xml version="1.0" encoding="UTF-8" ?><%s>foo</%s>' % (key, key)
+    try:
+        parseString(test_xml)
+        return True
+    except Exception: # minidom does not implement exceptions well
+        return False
+
+
+def make_valid_xml_name(key, attr):
+    """Tests an XML name and fixes it if invalid"""
+    LOG.info('Inside make_valid_xml_name(). Testing key "%s" with attr "%s"' % (
+        unicode_me(key), unicode_me(attr))
+             )
+    key = escape_xml(key)
+    attr = escape_xml(attr)
+
+    # pass through if key is already valid
+    if key_is_valid_xml(key):
+        return key, attr
+
+    # prepend a lowercase n if the key is numeric
+    if key.isdigit():
+        return 'n%s' % (key), attr
+
+    # replace spaces with underscores if that fixes the problem
+    if key_is_valid_xml(key.replace(' ', '_')):
+        return key.replace(' ', '_'), attr
+
+    # key is still invalid - move it into a name attribute
+    attr['name'] = key
+    key = 'key'
+    return key, attr
+
+
+def wrap_cdata(s):
+    """Wraps a string into CDATA sections"""
+    s = unicode_me(s).replace(']]>', ']]]]><![CDATA[>')
+    return '<![CDATA[' + s + ']]>'
+
+
+def default_item_func(parent):
+    return 'item'
+
+
+def convert(obj, ids, attr_type, item_func, cdata, parent='root'):
+    """Routes the elements of an object to the right function to convert them
+    based on their data type"""
+
+    LOG.info('Inside convert(). obj type is: "%s", obj="%s"' % (type(obj).__name__, unicode_me(obj)))
+
+    item_name = item_func(parent)
+
+    if isinstance(obj, numbers.Number) or type(obj) in (str, str):
+        return convert_kv(item_name, obj, attr_type, cdata)
+
+    if hasattr(obj, 'isoformat'):
+        return convert_kv(item_name, obj.isoformat(), attr_type, cdata)
+
+    if type(obj) == bool:
+        return convert_bool(item_name, obj, attr_type, cdata)
+
+    if obj is None:
+        return convert_none(item_name, '', attr_type, cdata)
+
+    if isinstance(obj, dict):
+        return convert_dict(obj, ids, parent, attr_type, item_func, cdata)
+
+    if isinstance(obj, collections.Iterable):
+        return convert_list(obj, ids, parent, attr_type, item_func, cdata)
+
+    raise TypeError('Unsupported data type: %s (%s)' % (obj, type(obj).__name__))
+
+
+def convert_dict(obj, ids, parent, attr_type, item_func, cdata):
+    """Converts a dict into an XML string."""
+    LOG.info('Inside convert_dict(): obj type is: "%s", obj="%s"' % (
+        type(obj).__name__, unicode_me(obj))
+             )
+    output = []
+    global expand_list_item_name
+    addline = output.append
+
+    item_name = item_func(parent)
+
+    for key, val in list(obj.items()):
+        LOG.info('Looping inside convert_dict(): key="%s", val="%s", type(val)="%s"' % (
+            unicode_me(key), unicode_me(val), type(val).__name__)
+                 )
+
+        attr = {} if not ids else {'id': '%s' % (get_unique_id(parent)) }
+
+        key, attr = make_valid_xml_name(key, attr)
+
+        if isinstance(val, numbers.Number) or type(val) in (str, str):
+            addline(convert_kv(key, val, attr_type, attr, cdata))
+
+        elif hasattr(val, 'isoformat'): # datetime
+            addline(convert_kv(key, val.isoformat(), attr_type, attr, cdata))
+
+        elif type(val) == bool:
+            addline(convert_bool(key, val, attr_type, attr, cdata))
+
+        elif isinstance(val, dict):
+            if attr_type:
+                attr['type'] = get_xml_type(val)
+            addline('<%s%s>%s</%s>' % (
+                key, make_attrstring(attr),
+                convert_dict(val, ids, key, attr_type, item_func, cdata),
+                key
+            )
+                    )
+
+        elif isinstance(val, collections.Iterable):
+            if attr_type:
+                attr['type'] = get_xml_type(val)
+            if expand_list_item_name:
+                addline('<%s%s>%s</%s>' % (
+                    key,
+                    make_attrstring(attr),
+                    convert_list(val, ids, key, attr_type, item_func, cdata),
+                    key
+                )
+                        )
+            else:
+                addline('%s' % convert_list(val, ids, key, attr_type, item_func, cdata))
+
+        elif val is None:
+            addline(convert_none(key, val, attr_type, attr, cdata))
+
+        else:
+            raise TypeError('Unsupported data type: %s (%s)' % (
+                val, type(val).__name__)
+                            )
+
+    return ''.join(output)
+
+
+def convert_list(items, ids, parent, attr_type, item_func, cdata):
+    """Converts a list into an XML string."""
+    LOG.info('Inside convert_list()')
+    global expand_list_item_name
+    output = []
+    addline = output.append
+
+    if expand_list_item_name:
+        item_name = item_func(parent)
+    else:
+        item_name = parent
+
+    if ids:
+        this_id = get_unique_id(parent)
+
+    for i, item in enumerate(items):
+        LOG.info('Looping inside convert_list(): item="%s", item_name="%s", type="%s"' % (
+            unicode_me(item), item_name, type(item).__name__)
+                 )
+        attr = {} if not ids else { 'id': '%s_%s' % (this_id, i+1) }
+        if isinstance(item, numbers.Number) or type(item) in (str, str):
+            addline(convert_kv(item_name, item, attr_type, attr, cdata))
+
+        elif hasattr(item, 'isoformat'): # datetime
+            addline(convert_kv(item_name, item.isoformat(), attr_type, attr, cdata))
+
+        elif type(item) == bool:
+            addline(convert_bool(item_name, item, attr_type, attr, cdata))
+
+        elif isinstance(item, dict):
+            if not attr_type:
+                addline('<%s>%s</%s>' % (
+                    item_name,
+                    convert_dict(item, ids, parent, attr_type, item_func, cdata),
+                    item_name,
+                )
+                        )
+            else:
+                addline('<%s type="dict">%s</%s>' % (
+                    item_name,
+                    convert_dict(item, ids, parent, attr_type, item_func, cdata),
+                    item_name,
+                )
+                        )
+
+        elif isinstance(item, collections.Iterable):
+            if not attr_type:
+                if expand_list_item_name:
+                    addline('<%s %s>%s</%s>' % (
+                        item_name, make_attrstring(attr),
+                        convert_list(item, ids, item_name, attr_type, item_func, cdata),
+                        item_name,
+                    ))
+                else:
+                    addline('%s' % convert_list(item, ids, item_name, attr_type, item_func, cdata))
+            else:
+                if expand_list_item_name:
+                    addline('<%s type="list"%s>%s</%s>' % (
+                        item_name, make_attrstring(attr),
+                        convert_list(item, ids, item_name, attr_type, item_func, cdata),
+                        item_name,
+                    )
+                            )
+                else:
+                    addline('%s' % convert_list(item, ids, item_name, attr_type, item_func, cdata))
+
+        elif item is None:
+            addline(convert_none(item_name, None, attr_type, attr, cdata))
+
+        else:
+            raise TypeError('Unsupported data type: %s (%s)' % (
+                item, type(item).__name__)
+                            )
+    return ''.join(output)
+
+
+def convert_kv(key, val, attr_type, attr={}, cdata=False):
+    """Converts a number or string into an XML element"""
+    LOG.info('Inside convert_kv(): key="%s", val="%s", type(val) is: "%s"' % (
+        unicode_me(key), unicode_me(val), type(val).__name__)
+             )
+
+    key, attr = make_valid_xml_name(key, attr)
+
+    if attr_type:
+        attr['type'] = get_xml_type(val)
+    attrstring = make_attrstring(attr)
+    return '<%s%s>%s</%s>' % (
+        key, attrstring,
+        wrap_cdata(val) if cdata == True else escape_xml(val),
+        key
+    )
+
+
+def convert_bool(key, val, attr_type, attr={}, cdata=False):
+    """Converts a boolean into an XML element"""
+    LOG.info('Inside convert_bool(): key="%s", val="%s", type(val) is: "%s"' % (
+        unicode_me(key), unicode_me(val), type(val).__name__)
+             )
+
+    key, attr = make_valid_xml_name(key, attr)
+
+    if attr_type:
+        attr['type'] = get_xml_type(val)
+    attrstring = make_attrstring(attr)
+    return '<%s%s>%s</%s>' % (key, attrstring, str(val).lower(), key)
+
+
+def convert_none(key, val, attr_type, attr={}, cdata=False):
+    """Converts a null value into an XML element"""
+    LOG.info('Inside convert_none(): key="%s"' % (unicode_me(key)))
+
+    key, attr = make_valid_xml_name(key, attr)
+
+    if attr_type:
+        attr['type'] = get_xml_type(val)
+    attrstring = make_attrstring(attr)
+    return '<%s%s></%s>' % (key, attrstring, key)
+
+
+def dicttoxml(obj, root=True, custom_root='root', ids=False, attr_type=True,
+              item_func=default_item_func, cdata=False, expand_list_item_name_flag=True):
+    """Converts a python object into XML.
+    Arguments:
+    - root specifies whether the output is wrapped in an XML root element
+      Default is True
+    - custom_root allows you to specify a custom root element.
+      Default is 'root'
+    - ids specifies whether elements get unique ids.
+      Default is False
+    - attr_type specifies whether elements get a data type attribute.
+      Default is True
+    - item_func specifies what function should generate the element name for
+      items in a list.
+      Default is 'item'
+    - cdata specifies whether string values should be wrapped in CDATA sections.
+      Default is False
+    """
+    LOG.info('Inside dicttoxml(): type(obj) is: "%s", obj="%s"' % (type(obj).__name__, unicode_me(obj)))
+    global expand_list_item_name
+    expand_list_item_name = expand_list_item_name_flag
+    output = []
+    addline = output.append
+    if root == True:
+        addline('<?xml version="1.0" encoding="UTF-8" ?>')
+        addline('<%s>%s</%s>' % (
+            custom_root,
+            convert(obj, ids, attr_type, item_func, cdata, parent=custom_root),
+            custom_root,
+        )
+                )
+    else:
+        addline(convert(obj, ids, attr_type, item_func, cdata, parent=''))
+    return ''.join(output).encode('utf-8')
+

toolkits/system/dict_helper.py

@@ -0,0 +1,29 @@
+# -*- coding: utf-8 -*-
+import argparse
+import sys
+from log4python.Log4python import log
+import traceback
+import importlib
+importlib.reload(sys)
+logger = log("MisUserActionEtl")
+
+
+def set_dict_val(dst_dict, key_list, val):
+    if len(key_list) > 0:
+        key = key_list[0]
+        del key_list[0]
+        dst_dict[key] = set_dict_val(dst_dict[key], key_list, val)
+        return dst_dict
+    else:
+        return val
+
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("logFile", type=str, help="specify the log file's path")
+        args = parser.parse_args()
+        print((args.logFile))
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())

toolkits/system/excel_helper.py

@@ -0,0 +1,101 @@
+# -*- coding: utf-8 -*-
+import argparse
+import sys
+
+import unicodecsv
+from log4python.Log4python import log
+import traceback
+import importlib
+importlib.reload(sys)
+logger = log("ExcelHelper")
+
+
+def write_cvs(column_names, rows, csv_file):
+    try:
+        with open(csv_file, 'wb') as f:
+            # w = unicodecsv.writer(f, encoding='utf-8-sig')
+            w = unicodecsv.writer(f, encoding='utf-8')
+            w.writerow(column_names)
+            w.writerows(rows)
+        logger.debug("CsvFile:%s" % csv_file)
+        ret = True
+    except Exception as ex:
+        ret = False
+        logger.error(traceback.format_exc())
+        logger.error('deal_msg error: %s' % ex)
+    return ret
+
+
+class ExcelHelper:
+    def __init__(self, file_path, column_split=","):
+        self.file_path = file_path
+        self.column_split = column_split
+        self.data = self.load_data()
+        self.cloumns_names = []
+
+    def load_data(self):
+        fp = open(self.file_path)
+        lines = fp.readlines()
+        if len(lines) == 0:
+            return []
+
+        self.cloumns_names = lines[0].strip("\n\r").split(self.column_split)
+        del lines[0]  # delete the title
+
+        data = []
+        for item in lines:
+            try:
+                columns_val = item.strip().split(self.column_split)
+                tmp_row = {}
+                for item_index in range(0, len(self.cloumns_names)):
+                    if item_index >= len(columns_val):
+                        val = ""
+                    else:
+                        val = columns_val[item_index]
+                    tmp_row[self.cloumns_names[item_index]] = val
+                data.append(tmp_row)
+            except Exception as ex:
+                logger.debug("Item:[%s]" % item)
+                logger.error(traceback.format_exc())
+                logger.error('deal_msg error: %s' % ex)
+        return data
+
+    def get_column_data(self, column_name):
+        return self.fetch_column_data(self.data, column_name)
+
+    @staticmethod
+    def fetch_column_data(data, column_name):
+        column_data = []
+        for item in data:
+            column_data.append(item[column_name])
+        return column_data
+
+    @staticmethod
+    def fetch_dict_data(data, key_column_name, val_columns_name):
+        data_dict = {}
+        for item in data:
+            tmp_val = {}
+            for val_name in val_columns_name:
+                tmp_val[val_name] = item[val_name]
+
+            key_name = str(item[key_column_name])
+            if key_name in dict(data_dict):
+                data_dict[key_name].append(tmp_val)
+            else:
+                tmp_list = [tmp_val]
+                data_dict[key_name] = tmp_list
+
+        return data_dict
+
+    def get_dict_data(self, key_column_name, val_columns_name):
+        return self.fetch_dict_data(self.data, key_column_name, val_columns_name)
+
+if __name__ == '__main__':
+    try:
+        parser = argparse.ArgumentParser()
+        parser.add_argument("excelFile", type=str, help="specify the excel file's path")
+        args = parser.parse_args()
+        csv = ExcelHelper(args.excelFile)
+    except Exception as ex:
+        logger.debug("Error: %s" % ex)
+        logger.debug(traceback.format_exc())

toolkits/system/file_helper.py

@@ -0,0 +1,52 @@
+# -*- coding: utf-8 -*-
+import re
+from os.path import basename
+from log4python.Log4python import log
+from unipath import Path
+logger = log("DirFileSearch")
+
+
+def read_content(file_path):
+    fp = open(file_path)
+    content = fp.readlines()
+    fp.close()
+    return content
+
+
+class DirFileSearch:
+    """:cvar
+    file_search = DirFileSearch(".*_1021.log")
+    file_compress_list = file_search.loop_filter_files(target_path)
+    """
+    def __init__(self, search_pattern):
+        self.__search_pattern = search_pattern
+
+    def __filter_by_name(self, file_path):
+        handle_path = Path(file_path)
+
+        if handle_path.isdir():
+            return file_path
+
+        if handle_path.isfile():
+            file_name = basename(file_path)
+            if re.findall(self.__search_pattern, file_name):
+                return file_path
+
+    def loop_filter_files(self, file_path):
+        file_list = []
+        handle_path = Path(file_path)
+
+        if handle_path.isdir():
+            search_path = file_path
+            loop_next_file_list = Path(search_path).listdir(filter=self.__filter_by_name)
+            for item in loop_next_file_list:
+                tmp_handle_path = Path(item)
+                if tmp_handle_path.isdir():
+                    tmp_file_list = self.loop_filter_files(item)
+                    file_list.extend(tmp_file_list)
+                else:
+                    file_list.append(item)
+
+        if handle_path.isfile():
+            file_list.append(file_path)
+        return file_list

toolkits/system/load_module.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+import imp
+import os
+import sys
+import traceback
+import uuid
+from log4python.Log4python import log
+import importlib
+importlib.reload(sys)
+logger = log("LoadModule")
+
+
+class LoadModule(object):
+    def __init__(self):
+        pass
+
+    @staticmethod
+    def load_from_file(file_path):
+        mod_name, file_ext = os.path.splitext(os.path.split(file_path)[-1])
+        py_mod = None
+        if file_ext.lower() == '.py':
+            py_mod = imp.load_source(mod_name, file_path)
+        elif file_ext.lower() == '.pyc':
+            py_mod = imp.load_compiled(mod_name, file_path)
+
+        return py_mod
+
+    @staticmethod
+    def __write_tmp_py(file_path, content):
+        fp = open(file_path, "w+")
+        fp.write(content)
+        fp.close()
+
+    def load_from_string(self, func_string, module_name=None):
+        modules = None
+        try:
+            str_random = str(uuid.uuid4()).replace("-", "")
+            if module_name:
+                file_path = "/tmp/dynamic_%s_%s.py" % (module_name, str_random)
+            else:
+                file_path = "/tmp/dynamic_%s.py" % str_random
+            self.__write_tmp_py(file_path, func_string)
+            modules = self.load_from_file(file_path)
+        except Exception as ex:
+            logger.debug("Error: %s" % ex)
+            logger.debug(traceback.format_exc())
+        return modules