wipo-gbd-transformation 1.1.105__py3-none-any.whl

gbdtransformation/parser.py

@@ -0,0 +1,190 @@
+import json
+import xmltodict
+import xml
+import os
+from munch import munchify, DefaultMunch
+import inspect
+from jinja2 import FileSystemLoader, Environment
+from gbdtransformation import renders
+from gbdtransformation.common import SilentUndefined, YAMLEverythingExtension
+from gbdvalidation.engine import RuleEngine
+import gzip
+
+
+def trim_nulls(value):
+    if hasattr(value, '__value'):
+        return trim_nulls(value['__value'])
+    return value if value is not None else ''
+
+
+class Parser:
+
+    """Class to parse input using the configuration render"""
+    def __init__(self, template_name, type=None):
+        """Constructor
+        @param template_name: the name of the template to be used
+        @param render: the render function that should be used
+        """
+
+        self.template = template_name
+        self.type = type
+
+        # fallback to guessing: it is used for migration and
+        # gbd-transform cli
+        if not self.type:
+            # brands special cases ugly hack
+            if template_name == 'woao':
+                self.type = 'brands'
+            elif template_name == 'wo6ter':
+                self.type = 'brands'
+            elif template_name == 'whoinn':
+                self.type = 'brands'
+            elif template_name.endswith('tm'):
+                self.type = 'brands'
+            elif template_name.endswith('id'):
+                self.type = 'designs'
+            elif template_name == 'emap' or template_name == 'emrp':
+                self.type = 'commons'
+            else:
+                self.type = 'common'
+        self.render = self.get_render()
+
+        templates_path = [
+            # brands/xxtm
+            os.path.join(os.path.dirname(__file__),
+                         self.type,
+                         self.template),
+
+            # brands | designs
+            os.path.join(os.path.dirname(__file__),
+                         self.type),
+
+            # common
+            os.path.join(os.path.dirname(__file__), 'common')
+        ]
+        for template_path in templates_path:
+            if not os.path.exists(template_path):
+                raise IOError("%s not found" % template_path)
+
+        # Create the jinja2 enviroment and load the general and template
+        # specific filter
+        ext = []
+        if self.render != 'XML':
+            ext = (YAMLEverythingExtension,)
+        self.env = Environment(autoescape=False,
+                               undefined=SilentUndefined,
+                               extensions=ext,
+                               lstrip_blocks=True,
+                               finalize=trim_nulls,
+                               loader=FileSystemLoader(
+                                   templates_path))
+        self.load_filters()
+
+    def load_data(self, input_file=None, input_string=None):
+        """
+        @param input_string: the input data as string
+        @param input_file: the input data as filename
+        """
+        # either one should be passed
+        if not((input_file is None) ^ (input_string is None)):
+            raise Exception(
+                "You must provide either an input_file or an input_string")
+
+        if input_file:
+            if input_file.endswith('.gz'):
+                with gzip.open(input_file, 'rb') as f:
+                    input_string = f.read()
+            else:
+                _, extension = os.path.splitext(input_file)
+                if extension not in ['.json', '.xml']:
+                    raise Exception('input_file can be only be JSON or XML')
+
+                with open(input_file, 'r') as f:
+                    input_string = ''.join(f.readlines())
+
+        # Decide if the input string is a JSON or XML
+        try:
+            data = json.loads(input_string)
+        except (json.decoder.JSONDecodeError, UnicodeDecodeError) as _:
+            try:
+                # namespaces to ignore
+                namespaces = {
+                    ns: None for ns in self.env.filters.get(
+                        'ignore_namespace', [])}
+                data = json.loads(
+                    json.dumps(xmltodict.parse(input_string,
+                                               process_namespaces=True,
+                                               namespaces=namespaces,
+                                               namespace_separator='_',
+                                               attr_prefix='_',
+                                               cdata_key='__value')))
+            except xml.parsers.expat.ExpatError as _:
+                raise Exception("Data can be only JSON or XML")
+        except Exception as e:
+            raise e
+        # Convert the python dict to python object for the input data
+        return munchify(data, factory=EmptyNoneMunch)
+
+
+    def load_filters(self):
+        """Jinja filter loader"""
+        modules = ['gbdtransformation.common.filters', 'gbdtransformation.%s.filters' % self.type,
+                   'gbdtransformation.%s.%s.filters' % (self.type, self.template)]
+        for tmp in modules:
+            try:
+                module = tmp.split('.')[-1]
+                module_tmp = __import__(tmp, globals(), locals(), [module])
+                filters_raw = [
+                    x for x in inspect.getmembers(module_tmp)
+                    if not str(x[0]).startswith('_')]
+                for filter in filters_raw:
+                    self.env.filters[filter[0]] = filter[1]
+            except ModuleNotFoundError as e:
+                print(e)
+                continue
+
+    def get_render(self):
+        module = 'gbdtransformation.%s.%s' % (self.type, self.template)
+        module_tmp = module.split('.')[-1]
+        module_tmp = __import__(module, globals(), locals(), [module_tmp])
+        return getattr(module_tmp, 'render')
+
+    def run(self, input_data, raise_errors=False):
+        """Excecutor function for the parser"""
+        if os.path.exists(input_data):
+            data = self.load_data(input_file=input_data)
+        else:
+            data = self.load_data(input_string=input_data)
+        template = self.env.get_template('template.yml')
+        res = template.render(data)
+        res = renders.sanitize(res)
+        render = getattr(renders, self.render)
+        if raise_errors:
+            return render(res, raise_errors=True)
+        return render(res)
+
+    def run_with_object(self, *args, **kwargs):
+        template = self.env.get_template('template.yml')
+        res = template.render(kwargs)
+        res = renders.sanitize(res)
+        render = getattr(renders, self.render)
+        return render(res)
+
+    def validate(self, input_data, gbd_format=None):
+        """Validates an input xml against GDB-Validation"""
+        if gbd_format is None:
+            gbd_format = self.run(input_data)
+        validator = RuleEngine()
+        validation_errors = validator.validate(input_string=gbd_format)
+        return gbd_format, validation_errors
+
+
+class EmptyNoneMunch(DefaultMunch):
+    """
+    A Munch that returns a None value for missing keys.
+    """
+
+    def __init__(self, *args, **kwargs):
+        default = None
+        super(DefaultMunch, self).__init__(*args, **kwargs)
+        self.__default__ = default

gbdtransformation/renders.py

@@ -0,0 +1,149 @@
+import yaml
+import json
+import re
+
+from yaml.reader import Reader
+from lxml import etree
+
+
+NoDatesSafeLoader = yaml.SafeLoader
+NoDatesSafeLoader.yaml_implicit_resolvers = {
+    k: [r
+        for r in v if r[0] != 'tag:yaml.org,2002:timestamp']
+    for k, v in NoDatesSafeLoader.yaml_implicit_resolvers.items()
+}
+
+def _strip_invalid(s):
+    res = ''
+    for x in s:
+        if Reader.NON_PRINTABLE.match(x):
+            # res += '\\x{:x}'.format(ord(x))
+            continue
+        res += x
+    return res
+
+
+def _remove_none(root):
+    """ Internal function to remove empty nodes (nodes with no data)
+    @param root: the root of the dict sub-tree"""
+    new_dict = {}
+    if isinstance(root, dict):
+        for k, v in root.items():
+            v = _remove_none(v)
+            if isinstance(v, list):
+                v = _remove_none(v)
+            if v is not None:
+                new_dict[k] = v
+    elif isinstance(root, list):
+        v = [_remove_none(a) for a in root if a is not None]
+        if len(v) > 0 and v[0] is not None:
+            new_dict = v
+    else:
+        return root if root else None
+    return new_dict or None
+
+
+def sanitize(input_data):
+    special_chars = {
+        '&': '&',
+    }
+    for char in special_chars.keys():
+        input_data = input_data.replace(char, special_chars[char])
+    # Restore already transformed
+    matched = re.search(r'&(\w)+;', input_data)
+    if matched:
+        restore = matched.group().replace('&', '&')
+        input_data = input_data.replace(matched.group(), restore)
+    lines = input_data.split('\n')
+    new_lines = [line for line in lines if len(line.strip())]
+    input_data = '\n'.join(new_lines)
+    return input_data
+
+
+def JSON(input_data, raise_errors=False):
+    """ Renders input data as JSON
+    @param input_data: The data to convert to json
+    @return JSON like string"""
+    exceptions = []
+
+    special_chars = {
+        '&': '&',
+    }
+    for char in special_chars.keys():
+        input_data = input_data.replace(char, special_chars[char])
+
+    try:
+        input_data = yaml.load(_strip_invalid(input_data), Loader=NoDatesSafeLoader)
+    except Exception as e:
+        exceptions.append(e)
+        try:
+            input_data = json.loads(input_data)
+            exceptions = []
+        except Exception as f:
+            exceptions.append(f)
+    for e in exceptions:
+        if raise_errors:
+            raise e
+        print(e)
+    input_data = _remove_none(input_data)
+    return json.dumps(input_data, indent=4)
+
+
+def YAML(input_data, raise_errors=False):
+    """ Renders input data as YAML
+    @param input_data: The data to convert to YAML
+    @return YAML like string"""
+    exceptions = []
+    try:
+        input_data = json.loads(input_data)
+    except Exception as e:
+        exceptions.append(e)
+        try:
+            input_data = yaml.load(_strip_invalid(input_data), Loader=NoDatesSafeLoader)
+            exceptions = []
+        except Exception as f:
+            exceptions.append(f)
+    for e in exceptions:
+        if raise_errors:
+            raise e
+        print(e)
+    input_data = _remove_none(input_data)
+    return yaml.dump(input_data)
+
+
+def _remove_empty_chids(root, prev_parent):
+    """ Internal function to remove empty nodes (nodes with no data)
+    @param root: the root of the xml sub-tree
+    @param prev_parent: the parent of the root"""
+    for chid in root.getchildren():
+        _remove_empty_chids(chid, root)
+    if not list(root.getchildren()) and str(root.text).strip() in ['None', '']:
+        prev_parent.remove(root)
+
+
+def XML(input_data):
+    """ Renders input data as XML
+    @param input_data: The data to convert to XML
+    @return XML like string"""
+    input_data = sanitize(input_data)
+    parser = etree.XMLParser(encoding='utf-8', recover=True)
+    root = etree.fromstring(input_data, parser=parser)
+    # Convert None text to empty nodes
+    for elem in root.iter():
+        for child in list(elem):
+            if child.text == 'None':
+                elem.remove(child)
+    # Remove empty nodes
+    _remove_empty_chids(root, None)
+    output_data = etree.tostring(root, encoding='utf8', method='xml')
+    output_data = output_data.decode('utf-8')
+    output_data = output_data.replace(">False<", ">false<").replace(">True<", ">true<")
+    lines = output_data.split('\n')
+    remove_lines = []
+    # Remove empty lines
+    for line in lines:
+        if line.strip() == '':
+            remove_lines.append(line)
+    for line in remove_lines:
+        lines.remove(line)
+    return '\n'.join(lines)