xml2db 0.9.0__tar.gz

xml2db-0.9.0/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 Commission de régulation de l'énergie
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

xml2db-0.9.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.1
+Name: xml2db
+Version: 0.9.0
+Summary: Import complex XML files to a relational database
+Author-email: Commission de régulation de l'énergie <opensource@cre.fr>
+Project-URL: Documentation, https://cre-dev.github.io/xml2db
+Project-URL: Repository, https://github.com/cre-dev/xml2db
+Project-URL: Issues page, https://github.com/cre-dev/xml2db/issues
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: sqlalchemy
+Requires-Dist: xmlschema
+Requires-Dist: lxml
+Requires-Dist: graphlib_backport; python_version < "3.9"
+Provides-Extra: docs
+Requires-Dist: mkdocs-material; extra == "docs"
+Requires-Dist: mkdocstrings[python]; extra == "docs"
+Provides-Extra: tests
+Requires-Dist: pytest; extra == "tests"
+
+# Xml2db
+
+`xml2db` is a Python package which allows loading XML data into a relational database. It is designed to handle complex 
+schemas which cannot be denormalized to a flat table, without any custom code.
+
+It builds a data model (i.e. a set of database tables linked with foreign keys relationships) based on a XSD schema and
+allows parsing and loading XML files into the database, and get them back to XML, if needed.
+
+It is as simple as:
+
+```python
+from xml2db import DataModel
+
+# Create a data model of tables with relations based on the XSD file
+data_model = DataModel(
+    xsd_file="path/to/file.xsd", 
+    connection_string="mssql+pyodbc://server/database?driver=ODBC+Driver+17+for+SQL+Server&trusted_connection=yes",
+)
+# Parse an XML file based on this XSD
+document = data_model.parse_xml(
+    xml_file="path/to/file.xml"
+)
+# Insert the document content into the database
+document.insert_into_target_tables()
+```
+
+The data model will adhere closely to the XSD schema, but `xml2db` will perform simplifications aimed at limiting the 
+complexity of the resulting data model and the storage footprint.
+
+The raw data loaded into the database can then be processed using [DBT](https://www.getdbt.com/), SQL views or 
+stored procedures aimed at extracting, correcting and formatting the data into more user-friendly tables.
+
+`xml2db` is developed and used at the [French energy regulation authority (CRE)](https://www.cre.fr/) to process XML 
+data, notably [REMIT data](https://www.acer.europa.eu/remit/data-collection). There, it handles batches of ~500 MB XML 
+files translating into a 20+ tables data model in the database.
+
+This package uses `sqlalchemy` to interact with the database, so it should work with different database backends. It has
+been tested against PostgreSQL and MS SQL Server. It currently does not work with SQLite. You may have to install 
+additional packages to connect to your database (e.g. `pyodbc` which is the default connector for MS SQL Server, or 
+`psycopg2` for PostgreSQL).  
+
+**Please read the [package documentation website](https://cre-dev.github.io/xml2db) for all the details!**
+
+## Installation
+
+The package can be installed, preferably in a virtual environment, using `pip`:
+
+``` bash
+pip install xml2db
+```
+
+## Testing
+
+Running the tests requires installing additional development dependencies, after cloning the repo, with:
+
+```bash
+pip install -e .[tests,docs]
+```
+
+Run all tests with the following command:
+
+```bash
+python -m pytest
+```
+
+Integration tests require write access to a MS SQL server database; the connection string is provided as an environment 
+variable `DB_STRING`. If you want to run only conversion tests that do not require a database you can run:
+
+```bash
+pytest -m "not dbtest"
+`````
+
+## Contributing
+
+Contributions are more than welcome, as well as bug reports, starting with the project's 
+[issue page](https://github.com/cre-dev/xml2db/issues).

xml2db-0.9.0/README.md

@@ -0,0 +1,76 @@
+# Xml2db
+
+`xml2db` is a Python package which allows loading XML data into a relational database. It is designed to handle complex 
+schemas which cannot be denormalized to a flat table, without any custom code.
+
+It builds a data model (i.e. a set of database tables linked with foreign keys relationships) based on a XSD schema and
+allows parsing and loading XML files into the database, and get them back to XML, if needed.
+
+It is as simple as:
+
+```python
+from xml2db import DataModel
+
+# Create a data model of tables with relations based on the XSD file
+data_model = DataModel(
+    xsd_file="path/to/file.xsd", 
+    connection_string="mssql+pyodbc://server/database?driver=ODBC+Driver+17+for+SQL+Server&trusted_connection=yes",
+)
+# Parse an XML file based on this XSD
+document = data_model.parse_xml(
+    xml_file="path/to/file.xml"
+)
+# Insert the document content into the database
+document.insert_into_target_tables()
+```
+
+The data model will adhere closely to the XSD schema, but `xml2db` will perform simplifications aimed at limiting the 
+complexity of the resulting data model and the storage footprint.
+
+The raw data loaded into the database can then be processed using [DBT](https://www.getdbt.com/), SQL views or 
+stored procedures aimed at extracting, correcting and formatting the data into more user-friendly tables.
+
+`xml2db` is developed and used at the [French energy regulation authority (CRE)](https://www.cre.fr/) to process XML 
+data, notably [REMIT data](https://www.acer.europa.eu/remit/data-collection). There, it handles batches of ~500 MB XML 
+files translating into a 20+ tables data model in the database.
+
+This package uses `sqlalchemy` to interact with the database, so it should work with different database backends. It has
+been tested against PostgreSQL and MS SQL Server. It currently does not work with SQLite. You may have to install 
+additional packages to connect to your database (e.g. `pyodbc` which is the default connector for MS SQL Server, or 
+`psycopg2` for PostgreSQL).  
+
+**Please read the [package documentation website](https://cre-dev.github.io/xml2db) for all the details!**
+
+## Installation
+
+The package can be installed, preferably in a virtual environment, using `pip`:
+
+``` bash
+pip install xml2db
+```
+
+## Testing
+
+Running the tests requires installing additional development dependencies, after cloning the repo, with:
+
+```bash
+pip install -e .[tests,docs]
+```
+
+Run all tests with the following command:
+
+```bash
+python -m pytest
+```
+
+Integration tests require write access to a MS SQL server database; the connection string is provided as an environment 
+variable `DB_STRING`. If you want to run only conversion tests that do not require a database you can run:
+
+```bash
+pytest -m "not dbtest"
+`````
+
+## Contributing
+
+Contributions are more than welcome, as well as bug reports, starting with the project's 
+[issue page](https://github.com/cre-dev/xml2db/issues).

xml2db-0.9.0/pyproject.toml

@@ -0,0 +1,39 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "xml2db"
+version = "0.9.0"
+authors = [
+  { name="Commission de régulation de l'énergie", email="opensource@cre.fr" },
+]
+description = "Import complex XML files to a relational database"
+readme = "README.md"
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "sqlalchemy",
+    "xmlschema",
+    "lxml",
+    "graphlib_backport;python_version<'3.9'",
+]
+
+[project.optional-dependencies]
+docs = ["mkdocs-material", "mkdocstrings[python]"]
+tests = ["pytest"]
+
+[project.urls]
+"Documentation" = "https://cre-dev.github.io/xml2db"
+"Repository" = "https://github.com/cre-dev/xml2db"
+"Issues page" = "https://github.com/cre-dev/xml2db/issues"
+
+[tool.pytest.ini_options]
+markers = [
+    "dbtest: marks tests as integration tests requiring a database backend (deselect with '-m \"not dbtest\"')",
+]
+junit_family = "xunit2"

xml2db-0.9.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

xml2db-0.9.0/src/debug.py

@@ -0,0 +1,34 @@
+import os
+from xml2db import DataModel
+from sqlalchemy import inspect
+
+from tests.sample_models.models import models
+
+
+def setup():
+    model_config = models[2]
+
+    model = DataModel(
+        os.path.join("../", model_config["xsd_path"]),
+        short_name="junit",
+        model_config=model_config["versions"][0]["config"],
+    )
+
+    return model
+
+
+def main():
+    from sqlalchemy import create_engine
+
+    connection_string = "mssql+pyodbc://DATACRE\DEV_BASECRE/BaseCRE?driver=ODBC+Driver+17+for+SQL+Server&trusted_connection=yes"
+
+    engine = create_engine(
+        "postgresql+psycopg2://testuser:testuser@localhost:5432/testdb"
+    )
+
+    inspector = inspect(engine)
+    print("ok")
+
+
+if __name__ == "__main__":
+    main()

xml2db-0.9.0/src/xml2db/__init__.py

@@ -0,0 +1,21 @@
+from xml2db.model import DataModel
+from xml2db.document import Document
+from xml2db.table import (
+    DataModelTable,
+    DataModelTableReused,
+    DataModelTableDuplicated,
+    DataModelColumn,
+    DataModelRelationN,
+    DataModelRelation1,
+)
+
+__all__ = [
+    "DataModel",
+    "Document",
+    "DataModelTable",
+    "DataModelTableReused",
+    "DataModelTableDuplicated",
+    "DataModelColumn",
+    "DataModelRelation1",
+    "DataModelRelationN",
+]