xml2db 0.9.0__py3-none-any.whl

xml2db/table/transformed_table.py

@@ -0,0 +1,314 @@
+from typing import Union, List, Tuple
+
+from xml2db.exceptions import DataModelConfigError
+from .column import DataModelColumn
+from .relations import DataModelRelation1, DataModelRelationN
+from .table import DataModelTable
+
+
+class DataModelTableTransformed(DataModelTable):
+    """A class extending DataModelTable with transformations
+
+    This class allows simplifying a DataModelTable object with default or configured transformations in \
+    order to reduce final schema complexity.
+    """
+
+    def _can_choice_transform_table(self) -> bool:
+        """Check if the table is of type "choice" and can be transformed to type/value fields.
+
+        :return: True if the table model be converted to type/value choice model, False otherwise
+        """
+        if self.model_group == "choice":
+            col_types = list(set([col.data_type for col in self.columns.values()]))
+            return (
+                len(self.relations_1) == 0
+                and len(self.relations_n) == 0
+                and len(col_types) == 1
+            )
+        return False
+
+    def _is_table_choice_transform_applicable(self) -> bool:
+        """Determine if choice transform should be applied to the whole table.
+
+        We try the choice_transform value provided in config, if any, and otherwise fall back to default value.
+
+        :return: True if choice transform is to be applied, False otherwise.
+        """
+        if "choice_transform" in self.config:
+            if isinstance(self.config["choice_transform"], bool):
+                if self.config["choice_transform"]:
+                    if self._can_choice_transform_table():
+                        return True
+                    else:
+                        raise DataModelConfigError(
+                            f"Choice-transform cannot be applied to table '{self.name}', see conditions in "
+                            f"DataModelTableTransformed._can_choice_transform_table."
+                        )
+                else:
+                    return False
+            else:
+                raise DataModelConfigError(
+                    f"Unrecognized choice_transform value '{self.config['choice_transform']}'"
+                    f" for table '{self.name}'. Only boolean values True or False are allowed."
+                )
+        elif self._can_choice_transform_table() and len(self.columns) > 2:
+            # column number isn't reduced if the number of columns = 2, as it would be elevated to 2 columns then
+            return True
+        return False
+
+    def _transform_to_choice(self) -> None:
+        """Transform the current table to a choice model representation with only type and value fields"""
+        col_types = list(set([col.data_type for col in self.columns.values()]))
+        col_names = [col.name for col in self.columns.values()]
+        min_lengths = [col.min_length for col in self.columns.values()]
+        max_lengths = [col.max_length for col in self.columns.values()]
+        allow_empty = [col.allow_empty for col in self.columns.values()]
+        self.columns = {
+            "type": DataModelColumn(
+                "type",
+                [("type", None)],
+                "string",
+                [1, 1],
+                min(len(name) for name in col_names),
+                max(len(name) for name in col_names),
+                False,
+                False,
+                False,
+                None,
+                self.config,
+                self.data_model,
+            ),
+            "value": DataModelColumn(
+                "value",
+                [("value", None)],
+                col_types[0],
+                [1, 1],
+                min(min_lengths) if all(e is not None for e in min_lengths) else None,
+                max(max_lengths) if all(e is not None for e in max_lengths) else None,
+                False,
+                False,
+                any(allow_empty),
+                None,
+                self.config,
+                self.data_model,
+            ),
+        }
+        self.fields = [
+            ("col", "type", self.columns["type"]),
+            ("col", "value", self.columns["value"]),
+        ]
+
+    def _can_transform_field(
+        self, field_type: str, field_name: str, transform: str = "join"
+    ) -> bool:
+        """Check if a given transformation can be applied to a given field
+
+        :param field_type: the field type ("col", "rel1" or "reln")
+        :param field_name: the field name
+        :param transform: the transform to be tested
+        :return: True is the field can be transformed
+        """
+        if field_type == "col" and transform == "join":
+            # check if simple columns with max occurrences > 1 can be joined as string
+            if self.columns[field_name].can_join_values_as_string:
+                return True
+        elif field_type == "rel1":
+            return transform in ["elevate", "elevate_wo_prefix"]
+        # "reln" can never be transformed
+        return False
+
+    def _get_field_transform(
+        self, field_type: str, field_name: str
+    ) -> Union[str, None]:
+        """Get the transformation that should be applied to this field, taking into account user-provided config
+
+        :param field_type: the field type ("col", "rel1", "reln")
+        :param field_name: the field name
+        :return: the default transformation that should be applied
+        """
+        field_config = self.config.get("fields", {}).get(field_name, {})
+        if "transform" in field_config:
+            if field_config["transform"] is False:
+                return None
+            if self._can_transform_field(
+                field_type, field_name, field_config["transform"]
+            ):
+                return field_config["transform"]
+            else:
+                raise DataModelConfigError(
+                    f"Transform value '{field_config['transform']}' cannot be applied"
+                    f" to field {field_name} of table '{self.name}'."
+                )
+        else:
+            if field_type == "col":
+                if self._can_transform_field("col", field_name, "join"):
+                    return "join"
+            elif field_type == "rel1":
+                if (
+                    self.relations_1[field_name].occurs[0] == 1
+                    or len(self.relations_1[field_name].other_table.columns) <= 4
+                ) and len(self.relations_1[field_name].other_table.parents_n) == 0:
+                    transform = (
+                        "elevate_wo_prefix"
+                        if len(self.columns) == 0 and len(self.relations_1) == 1
+                        else "elevate"
+                    )
+                    if self._can_transform_field("rel1", field_name, transform):
+                        return transform
+
+    def _elevate_relation_1(
+        self, rel_name, transform
+    ) -> List[
+        Tuple[str, str, Union[DataModelColumn, DataModelRelation1, DataModelRelationN]]
+    ]:
+        """Elevate a child table to the upper level"""
+        rel = self.relations_1[rel_name]
+        if transform == "elevate_wo_prefix":
+            prefix = ""
+        else:
+            prefix = f"{rel.name}_"
+
+        del self.relations_1[rel_name]
+
+        # insert the children fields into the current table
+        elevated_fields = []
+        for child_field_type, key, child_field in rel.other_table.fields:
+            prefixed_key = f"{prefix}{key}"
+            if child_field_type == "col":
+                self.columns[prefixed_key] = DataModelColumn(
+                    prefixed_key,
+                    [(rel.name, rel.other_table.type_name)] + child_field.name_chain,
+                    child_field.data_type,
+                    [0, child_field.occurs[1]]
+                    if rel.occurs[0] == 0
+                    else child_field.occurs,
+                    child_field.min_length,
+                    child_field.max_length,
+                    child_field.is_attr,
+                    child_field.is_content,
+                    child_field.allow_empty,
+                    child_field.ngroup,
+                    self.config,
+                    self.data_model,
+                )
+                elevated_fields.append(
+                    (
+                        "col",
+                        prefixed_key,
+                        self.columns[prefixed_key],
+                    )
+                )
+            elif child_field_type == "rel1":
+                self.relations_1[prefixed_key] = DataModelRelation1(
+                    prefixed_key,
+                    [(rel.name, rel.other_table.type_name)] + child_field.name_chain,
+                    self,
+                    child_field.other_table,
+                    [0, child_field.occurs[1]]
+                    if rel.occurs[0] == 0
+                    else child_field.occurs,
+                    child_field.ngroup,
+                    self.data_model,
+                )
+                elevated_fields.append(
+                    (
+                        "rel1",
+                        prefixed_key,
+                        self.relations_1[prefixed_key],
+                    )
+                )
+            elif child_field_type == "reln":
+                self.relations_n[prefixed_key] = DataModelRelationN(
+                    prefixed_key,
+                    [(rel.name, rel.other_table.type_name)] + child_field.name_chain,
+                    self,
+                    child_field.other_table,
+                    [0, child_field.occurs[1]]
+                    if rel.occurs[0] == 0
+                    else child_field.occurs,
+                    child_field.ngroup,
+                    self.data_model,
+                )
+                elevated_fields.append(
+                    (
+                        "reln",
+                        prefixed_key,
+                        self.relations_n[prefixed_key],
+                    )
+                )
+        return elevated_fields
+
+    def simplify_table(self) -> Tuple[dict, dict]:
+        """Simplify table recursively and return a dict of simplifications applied
+
+        Return values are dict which associate xml types and xml field with transform operations. These dicts are used \
+        at parsing stage and should contain all xml types and field names, even if there is no transformation to apply.\
+        Transformations are described by keywords:
+           For tables (aka XML complex types):
+              - "choice_transform": transform a choice between 2+ different fields to type / value fields in order to \
+              reduce the number of columns.
+           For fields:
+              - "join": applies to fields with multiple values allowed: append all values in a comma separated string
+              - "elevate": pull up child type to parent level, appending field name to child field names
+              - "elevate_wo_prefix": same as "elevate" but keeping only child's fields names (without prefixing)
+              - False: prevents any transformation on this field
+
+
+        :return: a tuple of 2 dicts: the first one for type transforms, the second one for fields transforms
+        """
+
+        # if the table is already simplified, stop here
+        if self.is_simplified:
+            return {}, {}
+        self.is_simplified = True
+
+        # if the table can be transformed, stop here
+        if self._is_table_choice_transform_applicable():
+            self._transform_to_choice()
+            self.is_simplified = True
+            return {self.type_name: "choice"}, {}
+
+        # loop through field to transform them if need be
+        out_fields = []
+        types_transforms = {}
+        fields_transforms = {}
+        for field_type, field_name, field in self.fields:
+            if field_type == "col":
+                if self._get_field_transform("col", field_name) == "join":
+                    fields_transforms[(self.type_name, field_name)] = (
+                        None,
+                        "join",
+                    )
+                out_fields.append(("col", field_name, field))
+
+            else:
+                # simplify child table
+                (
+                    types_transforms_child,
+                    fields_transforms_child,
+                ) = field.other_table.simplify_table()
+                types_transforms.update(types_transforms_child)
+                fields_transforms.update(fields_transforms_child)
+
+                # check if children can be "elevated" to the upper level
+                transform = self._get_field_transform(field_type, field_name)
+                if transform is not None:
+                    if field_type == "rel1":
+                        elevated_fields = self._elevate_relation_1(
+                            field_name, transform
+                        )
+                        out_fields.extend(elevated_fields)
+                        fields_transforms[(self.type_name, field_name)] = (
+                            field.other_table.type_name,
+                            transform,
+                        )
+                else:
+                    out_fields.append((field_type, field_name, field))
+                    fields_transforms[(self.type_name, field_name)] = (
+                        field.other_table.type_name,
+                        None,
+                    )
+                    field.other_table.keep_table = True
+
+        self.fields = out_fields
+        return types_transforms, fields_transforms

xml2db/xml_converter.py

@@ -0,0 +1,258 @@
+import typing
+from datetime import datetime
+from typing import Union
+import logging
+from lxml import etree
+from io import BytesIO
+from itertools import zip_longest
+
+if typing.TYPE_CHECKING:
+    from xml2db.model import DataModel
+
+logger = logging.getLogger(__name__)
+
+
+class XMLConverter:
+    def __init__(self, data_model: "DataModel", document_tree: dict = None):
+        """A class to convert data from document tree format (nested dict) to and from XML.
+
+        :param data_model: The `DataModel` object
+        :param document_tree: Data in the document tree format (optional, can be built by `parse_xml` function)
+        """
+        self.model = data_model
+        self.document_tree = document_tree
+
+    def parse_xml(
+        self,
+        xml_file: Union[str, BytesIO],
+        file_path: str = None,
+        skip_validation: bool = False,
+    ) -> dict:
+        """Parse an XML document into a nested dict and performs the simplifications defined in the
+        DataModel object ("pull" child to upper level, transform a choice model into "type" and "value"
+        fields or concatenate children as string).
+
+        :param xml_file: An XML file path or file content to be converted
+        :param file_path: The file path to be printed in logs
+        :param skip_validation: Whether we should validate XML against the schema before parsing
+        :return: The parsed data in the document tree format (nested dict)
+        """
+        logger.info(f"Parsing XML file: {file_path}")
+
+        xt = etree.parse(xml_file)
+        if skip_validation:
+            logger.info("Skipping XML file validation")
+        else:
+            logger.info("Validating XML file against the schema")
+            if not self.model.xml_schema.is_valid(xt):
+                logger.error(f"XML file {file_path} does not conform with the schema")
+                raise ValueError(
+                    f"XML file {file_path} does not conform with the schema"
+                )
+            logger.info("XML file conforms with the schema")
+
+        if self.model.tables[self.model.root_table].is_virtual_node:
+            doc = etree.Element(self.model.root_table)
+            doc.append(xt.getroot())
+        else:
+            doc = xt.getroot()
+        self.document_tree = self._parse_xml_node(self.model.root_table, doc)
+        return self.document_tree
+
+    def _parse_xml_node(self, node_type: str, node: etree.Element) -> dict:
+        """Parse nodes of an XML document into a dict recursively
+
+        This method is much faster than using xmlschema parse method, but it will not
+        check the validity of the document regarding the XSD. It expects to
+        deal with a valid XML document.
+
+        :param node_type: type of the node
+        :param node: lxml node object
+        :return: a dict representing the node content
+        """
+
+        result = {"type": node_type, "content": {}}
+
+        for key, val in node.attrib.items():
+            if (
+                key
+                != "{http://www.w3.org/2001/XMLSchema-instance}noNamespaceSchemaLocation"
+            ):
+                result["content"][key] = [val]
+
+        if node.text and node.text.strip():
+            result["content"]["value"] = [node.text.strip()]
+
+        for element in node.iterchildren():
+            key = element.tag.split("}")[1] if "}" in element.tag else element.tag
+            node_type_key = (node_type, key)
+            value = None
+            if element.text and element.text.strip():
+                value = element.text
+            if (
+                self.model.fields_transforms.get(node_type_key, (None, "join"))[1]
+                != "join"
+            ):
+                value = self._parse_xml_node(
+                    self.model.fields_transforms[node_type_key][0], element
+                )
+
+            if key in result["content"]:
+                result["content"][key].append(value)
+            else:
+                result["content"][key] = [value]
+
+        for key in list(result["content"]):
+            node_type_key = (node_type, key)
+            if node_type_key in self.model.fields_transforms:
+                transform = self.model.fields_transforms[node_type_key][1]
+                if transform == "elevate" or transform == "elevate_wo_prefix":
+                    prefix = f"{key}_" if transform == "elevate" else ""
+                    child = result["content"][key][0]
+                    child_content = child["content"]
+                    del result["content"][key]
+                    for child_key, val in child_content.items():
+                        result["content"][f"{prefix}{child_key}"] = val
+
+        if node_type in self.model.types_transforms:
+            if self.model.types_transforms[node_type] == "choice":
+                result["content"] = [
+                    {"type": [child_key], "value": val}
+                    for child_key, val in result["content"].items()
+                ][0]
+
+        return result
+
+    def to_xml(
+        self, out_file: str = None, nsmap: dict = None, indent: str = "  "
+    ) -> etree.Element:
+        """Convert a document tree (nested dict) into an XML file
+
+        :param out_file: If provided, write output to a file.
+        :param nsmap: An optional namespace mapping.
+        :param indent: A string used as indentin XML output.
+        :return: The etree object corresponding to the root XML node.
+        """
+        doc = self._make_xml_node(
+            self.document_tree,
+            self.model.tables[self.document_tree["type"]].name,
+            nsmap,
+        )
+        if self.model.tables[self.model.root_table].is_virtual_node:
+            child = None
+            for child in doc:
+                break
+            doc = child
+        if out_file:
+            etree.indent(doc, space=indent)
+            with open(out_file, "wt") as f:
+                f.write(
+                    etree.tostring(
+                        doc,
+                        pretty_print=True,
+                        encoding="utf-8",
+                        xml_declaration=True,
+                    ).decode("utf-8")
+                )
+        return doc
+
+    def _make_xml_node(self, node_data, node_name, nsmap: dict = None):
+        def check_transformed_node(node_type, element):
+            if (
+                node_type in self.model.types_transforms
+                and self.model.types_transforms[node_type] == "choice"
+            ):
+                new_node = etree.Element(element.tag)
+                extracted = {}
+                for child in element:
+                    extracted[child.tag] = child.text
+                if "type" in extracted and "value" in extracted:
+                    child_node = etree.Element(extracted["type"])
+                    child_node.text = extracted["value"]
+                    new_node.append(child_node)
+                    return new_node
+                else:
+                    return None
+            return element
+
+        tb = self.model.tables[node_data["type"]]
+        nodes_stack = [(node_data["type"], etree.Element(node_name, nsmap=nsmap))]
+        prev_chain = []
+        prev_ngroup = None
+        ngroup_stack = []
+        for field_type, rel_name, rel in tb.fields:
+            name_chain = rel.name_chain[:-1]
+            i = len(prev_chain)
+            while i > 0 and (
+                i > len(name_chain) or name_chain[i - 1][0] != prev_chain[i - 1][0]
+            ):
+                completed_node = check_transformed_node(*nodes_stack.pop())
+                if completed_node is not None and len(completed_node) > 0:
+                    nodes_stack[-1][1].append(completed_node)
+                i -= 1
+            while i < len(name_chain):
+                node = etree.Element(name_chain[i][0])
+                nodes_stack.append(
+                    (
+                        name_chain[i][1],
+                        node,
+                    )
+                )
+                i += 1
+            prev_chain = name_chain
+            children = []
+            attributes = {}
+            text_content = None
+            if field_type == "col":
+                if rel_name in node_data["content"]:
+                    if rel.is_attr:
+                        attributes[rel.name_chain[-1][0]] = node_data["content"][
+                            rel_name
+                        ][0]
+                    elif rel.is_content:
+                        text_content = node_data["content"][rel_name][0]
+                    else:
+                        for field_value in node_data["content"][rel_name]:
+                            child = etree.Element(rel.name_chain[-1][0])
+                            if isinstance(field_value, datetime):
+                                field_value = field_value.isoformat()
+                            child.text = str(field_value).encode("utf-8")
+                            children.append(child)
+            elif field_type == "rel1":
+                if rel_name in node_data["content"]:
+                    child = self._make_xml_node(
+                        node_data["content"][rel_name][0], rel.name_chain[-1][0]
+                    )
+                    children = [child]
+            elif field_type == "reln":
+                if rel_name in node_data["content"]:
+                    children = [
+                        self._make_xml_node(child_tree, rel.name_chain[-1][0])
+                        for child_tree in node_data["content"][rel_name]
+                    ]
+            if prev_ngroup and rel.ngroup != prev_ngroup:
+                for ngroup_children in zip_longest(*ngroup_stack):
+                    for child in ngroup_children:
+                        nodes_stack[-1][1].append(child)
+                ngroup_stack = []
+            prev_ngroup = rel.ngroup
+            if len(children) > 0:
+                if rel.ngroup:
+                    ngroup_stack.append(children)
+                else:
+                    for child in children:
+                        nodes_stack[-1][1].append(child)
+            for key, val in attributes.items():
+                nodes_stack[-1][1].set(key, val)
+            if text_content is not None:
+                nodes_stack[-1][1].text = text_content
+        if len(ngroup_stack) > 0:
+            for ngroup_children in zip_longest(*ngroup_stack):
+                for child in ngroup_children:
+                    nodes_stack[-1][1].append(child)
+        while len(nodes_stack) > 1:
+            node = check_transformed_node(*nodes_stack.pop())
+            if node is not None and len(node) > 0:
+                nodes_stack[-1][1].append(node)
+
+        return check_transformed_node(*nodes_stack[0])

xml2db-0.9.0.dist-info/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Commission de régulation de l'énergie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

xml2db-0.9.0.dist-info/METADATA

@@ -0,0 +1,100 @@
+Metadata-Version: 2.1
+Name: xml2db
+Version: 0.9.0
+Summary: Import complex XML files to a relational database
+Author-email: Commission de régulation de l'énergie <opensource@cre.fr>
+Project-URL: Documentation, https://cre-dev.github.io/xml2db
+Project-URL: Repository, https://github.com/cre-dev/xml2db
+Project-URL: Issues page, https://github.com/cre-dev/xml2db/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy
+Requires-Dist: xmlschema
+Requires-Dist: lxml
+Requires-Dist: graphlib-backport ; python_version < "3.9"
+Provides-Extra: docs
+Requires-Dist: mkdocs-material ; extra == 'docs'
+Requires-Dist: mkdocstrings[python] ; extra == 'docs'
+Provides-Extra: tests
+Requires-Dist: pytest ; extra == 'tests'
+
+# Xml2db
+
+`xml2db` is a Python package which allows loading XML data into a relational database. It is designed to handle complex 
+schemas which cannot be denormalized to a flat table, without any custom code.
+
+It builds a data model (i.e. a set of database tables linked with foreign keys relationships) based on a XSD schema and
+allows parsing and loading XML files into the database, and get them back to XML, if needed.
+
+It is as simple as:
+
+```python
+from xml2db import DataModel
+
+# Create a data model of tables with relations based on the XSD file
+data_model = DataModel(
+    xsd_file="path/to/file.xsd", 
+    connection_string="mssql+pyodbc://server/database?driver=ODBC+Driver+17+for+SQL+Server&trusted_connection=yes",
+)
+# Parse an XML file based on this XSD
+document = data_model.parse_xml(
+    xml_file="path/to/file.xml"
+)
+# Insert the document content into the database
+document.insert_into_target_tables()
+```
+
+The data model will adhere closely to the XSD schema, but `xml2db` will perform simplifications aimed at limiting the 
+complexity of the resulting data model and the storage footprint.
+
+The raw data loaded into the database can then be processed using [DBT](https://www.getdbt.com/), SQL views or 
+stored procedures aimed at extracting, correcting and formatting the data into more user-friendly tables.
+
+`xml2db` is developed and used at the [French energy regulation authority (CRE)](https://www.cre.fr/) to process XML 
+data, notably [REMIT data](https://www.acer.europa.eu/remit/data-collection). There, it handles batches of ~500 MB XML 
+files translating into a 20+ tables data model in the database.
+
+This package uses `sqlalchemy` to interact with the database, so it should work with different database backends. It has
+been tested against PostgreSQL and MS SQL Server. It currently does not work with SQLite. You may have to install 
+additional packages to connect to your database (e.g. `pyodbc` which is the default connector for MS SQL Server, or 
+`psycopg2` for PostgreSQL).  
+
+**Please read the [package documentation website](https://cre-dev.github.io/xml2db) for all the details!**
+
+## Installation
+
+The package can be installed, preferably in a virtual environment, using `pip`:
+
+``` bash
+pip install xml2db
+```
+
+## Testing
+
+Running the tests requires installing additional development dependencies, after cloning the repo, with:
+
+```bash
+pip install -e .[tests,docs]
+```
+
+Run all tests with the following command:
+
+```bash
+python -m pytest
+```
+
+Integration tests require write access to a MS SQL server database; the connection string is provided as an environment 
+variable `DB_STRING`. If you want to run only conversion tests that do not require a database you can run:
+
+```bash
+pytest -m "not dbtest"
+`````
+
+## Contributing
+
+Contributions are more than welcome, as well as bug reports, starting with the project's 
+[issue page](https://github.com/cre-dev/xml2db/issues).