xplan-tools 1.11.1__tar.gz → 1.12.0__tar.gz

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/PKG-INFO

@@ -1,8 +1,9 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: xplan-tools
-Version: 1.11.1
+Version: 1.12.0
 Summary: Manage XPlanung data
 License: EUPL-1.2-or-later
+License-File: LICENSE.md
 Author: Tobias Kraft
 Author-email: tobias.kraft@gv.hamburg.de
 Requires-Python: >=3.10,<3.14

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "xplan-tools"
-version = "1.11.1"
+version = "1.12.0"
 description = "Manage XPlanung data"
 authors = [
     { name = "Tobias Kraft", email = "tobias.kraft@gv.hamburg.de" },
@@ -84,7 +84,7 @@ markers = [
 ]
 
 [tool.coverage.run]
-omit = ["tests/*", "xplan_tools/interface/base.py"]
+omit = ["tests/*", "xplan_tools/interface/base.py",  "xplan_tools/model/migrations/*"]
 
 [build-system]
 requires = ["poetry-core"]

xplan_tools-1.12.0/xplan_tools/interface/db.py

@@ -0,0 +1,398 @@
+"""Module containing the class for extracting plans from and writing to databases."""
+
+# import json
+import logging
+from pathlib import Path
+from typing import Iterable
+
+from alembic import command, config, script
+from geoalchemy2 import load_spatialite_gpkg
+from geoalchemy2.admin.dialects.sqlite import load_spatialite_driver
+from sqlalchemy import (
+    Column,
+    Engine,
+    MetaData,
+    Table,
+    create_engine,
+    delete,
+    insert,
+    inspect,
+    select,
+    text,
+)
+from sqlalchemy.engine import URL, make_url
+
+# from sqlalchemy.dialects.sqlite.base import SQLiteCompiler
+from sqlalchemy.event import listen, listens_for, remove
+
+# from sqlalchemy.ext.compiler import compiles
+from sqlalchemy.orm import sessionmaker
+
+# from sqlalchemy.sql.expression import BindParameter
+from xplan_tools.model import model_factory
+from xplan_tools.model.base import BaseCollection, BaseFeature
+from xplan_tools.model.orm import Base, Feature, Geometry, Refs
+from xplan_tools.util import check_schema_accessibility
+
+# from xplan_tools.util import linearize_geom
+from .base import BaseRepository
+
+logger = logging.getLogger(__name__)
+
+
+class DBRepository(BaseRepository):
+    """Repository class for loading from and writing to databases."""
+
+    def __init__(
+        self,
+        datasource: str = "",
+        schema: str | None = None,
+        srid: int = 25832,
+        with_views: bool = False,
+    ) -> None:
+        """Initializes the DB Repository.
+
+        During initialization, a connection is established and the existence of required tables is tested.
+        If an alembic revision is found, automatic migration is executed for PostgreSQL DBs.
+        For other DBs, an Exception is raised if the revision does not correspond to the current model.
+        If no revision and tables are found, they are automatically created.
+
+        Args:
+            datasource: A connection string which will be transformed to a URL instance.
+            schema: Schema name for DB repository. If not specified, the default schema is used. Only for PostgreSQL.
+            srid: the EPSG code for spatial data
+            with_views: whether to create geometrytype-specific views (postgres only)
+        """
+        self.datasource: URL = make_url(datasource)
+        self.content = None
+        self.schema = schema
+        self.dialect = self.datasource.get_dialect().name
+        self.Session = sessionmaker(bind=self._engine)
+        # self.session = self.Session()
+        self.srid = srid
+        self.with_views = with_views
+
+        self.alembic_cfg = config.Config()
+        self.alembic_cfg.set_main_option(
+            "script_location", "xplan_tools:model:migrations"
+        )
+        self.alembic_cfg.set_main_option("srid", str(srid))
+        if with_views:
+            self.alembic_cfg.set_main_option("with_views", "1")
+        self.alembic_cfg.set_main_option(
+            "sqlalchemy.url",
+            datasource.replace("gpkg:", "sqlite:").replace(
+                "postgresql:", "postgresql+psycopg:"
+            ),
+        )
+        if self.schema and self.dialect == "postgresql":
+            check_schema_accessibility(self._engine, self.schema)
+            self.alembic_cfg.set_main_option("custom_schema", self.schema)
+        current_version = script.ScriptDirectory.from_config(
+            self.alembic_cfg
+        ).get_heads()
+        # test for tables and revision
+        with self._engine.connect() as conn:
+            inspector = inspect(conn)
+            tables = inspector.get_table_names(schema=self.schema)
+            is_coretable = {"coretable", "refs"}.issubset(set(tables))
+            if "alembic_version" in tables:
+                alembic_table = Table(
+                    "alembic_version",
+                    MetaData(schema=self.schema),
+                    Column("version_num"),
+                )
+                stmt = select(alembic_table.c.version_num)
+                db_version = conn.execute(stmt).scalars().all()
+            else:
+                db_version = []
+            is_current_version = set(db_version) == set(current_version)
+            if is_current_version:
+                logger.info("Database is at current revision")
+                return
+        # handle schema upgrade or table creation
+        if is_coretable and not db_version:
+            e = RuntimeError("Coretable with no revision found in database")
+            e.add_note(
+                "it is likely that the database was set up with an older version of this library which didn't use revisions yet"
+            )
+            e.add_note(
+                "please set up a new database or add a revision corresponding to the current model manually"
+            )
+            raise e
+        # if postgresql, run alembic and return
+        elif self.dialect == "postgresql":
+            logger.info(
+                "Running database migrations"
+                if db_version
+                else "Creating new database schema"
+            )
+            command.upgrade(self.alembic_cfg, "head")
+            return
+        elif db_version:
+            e = NotImplementedError(
+                f"Incompatible database revision and automatic migration not implemented for {self.dialect}"
+            )
+            e.add_note(
+                "please set up a new database with the current version of this library"
+            )
+            raise e
+        else:
+            # create tables if it's a fresh file-based DB and set it to current revision
+            logger.info("Creating new database schema")
+            self.create_tables(self.srid)
+            command.stamp(self.alembic_cfg, "head")
+
+    @property
+    def _engine(self) -> Engine:
+        url = (
+            self.datasource.set(drivername="postgresql+psycopg")
+            if self.dialect == "postgresql"
+            else self.datasource
+        )
+        connect_args: dict[str, str] = {}
+        if self.schema and self.dialect == "postgresql":
+            connect_args["options"] = f"-csearch_path={self.schema},public"
+        engine = create_engine(url, connect_args=connect_args)
+        if self.dialect == "geopackage":
+            listen(engine, "connect", load_spatialite_gpkg)
+        elif self.dialect == "sqlite":
+            listen(
+                engine,
+                "connect",
+                load_spatialite_driver,
+            )
+        return engine
+
+    def get_plan_by_id(self, id: str) -> BaseCollection:
+        logger.debug(f"retrieving plan with id {id}")
+        with self.Session() as session:
+            plan_feature = session.get(Feature, id)
+            if not plan_feature:
+                raise ValueError(f"no feature found with id {id}")
+            elif "Plan" not in plan_feature.featuretype:
+                raise ValueError(f"{plan_feature.featuretype} is not a plan object")
+            else:
+                plan_model = model_factory(
+                    plan_feature.featuretype,
+                    plan_feature.version,
+                    plan_feature.appschema,
+                ).model_validate(plan_feature)
+                collection = {id: plan_model}
+                srid = plan_model.get_geom_srid()
+                # iterate related features with depth=2: plan -> section -> features
+                for feature in plan_feature.related_features(session, depth=2):
+                    collection[str(feature.id)] = model_factory(
+                        feature.featuretype, feature.version, feature.appschema
+                    ).model_validate(feature)
+                return BaseCollection(
+                    features=collection,
+                    srid=srid,
+                    version=plan_feature.version,
+                    appschema=plan_feature.appschema,
+                )
+
+    def get(self, id: str) -> BaseFeature:
+        logger.debug(f"retrieving feature with id {id}")
+        with self.Session() as session:
+            feature = session.get(Feature, id)
+            if not feature:
+                raise ValueError(f"no feature found with id {id}")
+            else:
+                return model_factory(
+                    feature.featuretype, feature.version, feature.appschema
+                ).model_validate(feature)
+
+    def save(self, feature: BaseFeature) -> None:
+        logger.debug(f"saving feature with id {id}")
+        with self.Session() as session:
+            feature = feature.model_dump_coretable()
+            if session.get(Feature, feature.id):
+                raise ValueError(f"feature with id {feature.id} already exists")
+            session.merge(feature)
+            session.commit()
+
+    def delete_plan_by_id(self, id: str) -> BaseFeature:
+        logger.debug(f"deleting plan with id {id}")
+        with self.Session() as session:
+            plan_feature = session.get(Feature, id)
+            if not plan_feature:
+                raise ValueError(f"no feature found with id {id}")
+            elif "Plan" not in plan_feature.featuretype:
+                raise ValueError(f"{plan_feature.featuretype} is not a plan object")
+            else:
+                plan_model = model_factory(
+                    plan_feature.featuretype,
+                    plan_feature.version,
+                    plan_feature.appschema,
+                ).model_validate(plan_feature)
+                ids = [plan_feature.id]
+                ids += [
+                    feature.id for feature in plan_feature.related_features(session)
+                ]
+                stmt = delete(Feature).where(Feature.id.in_(ids))
+                session.execute(stmt)
+                session.commit()
+                return plan_model
+
+    def delete(self, id: str) -> BaseFeature:
+        logger.debug(f"deleting feature with id {id}")
+        with self.Session() as session:
+            feature = session.get(Feature, id)
+            if not feature:
+                raise ValueError(f"no feature found with id {id}")
+            else:
+                session.delete(feature)
+                session.commit()
+                return model_factory(
+                    feature.featuretype, feature.version, feature.appschema
+                ).model_validate(feature)
+
+    def save_all(
+        self, features: BaseCollection | Iterable[BaseFeature], **kwargs
+    ) -> None:
+        logger.debug("saving collection")
+        with self.Session() as session:
+            feature_list = []
+            refs_list = []
+            for feature in (
+                features.get_features()
+                if isinstance(features, BaseCollection)
+                else features
+            ):
+                feature, refs = feature.model_dump_coretable_bulk()
+                feature_list.append(feature)
+                refs_list.extend([ref for ref in refs if ref not in refs_list])
+            if feature_list:
+                session.execute(insert(Feature), feature_list)
+            if refs_list:
+                session.execute(insert(Refs), refs_list)
+            session.commit()
+
+    def update_all(
+        self, features: BaseCollection | Iterable[BaseFeature], **kwargs
+    ) -> None:
+        logger.debug("updating collection")
+        with self.Session() as session:
+            for feature in (
+                features.get_features()
+                if isinstance(features, BaseCollection)
+                else features
+            ):
+                feature = feature.model_dump_coretable()
+                session.merge(feature)
+            session.commit()
+
+    def update(self, id: str, feature: BaseFeature) -> BaseFeature:
+        logger.debug(f"updating feature with id {id}")
+        with self.Session() as session:
+            db_feature = session.get(Feature, id)
+            if db_feature:
+                session.merge(feature.model_dump_coretable())
+                session.commit()
+                return feature
+            else:
+                raise ValueError(f"no feature found with id {id}")
+
+    def patch(self, id: str, partial_update: dict) -> BaseFeature:
+        logger.debug(f"patching feature with id {id}: {partial_update}")
+        with self.Session() as session:
+            db_feature = session.get(Feature, id)
+            if db_feature:
+                feature_dict = (
+                    model_factory(
+                        db_feature.featuretype, db_feature.version, db_feature.appschema
+                    )
+                    .model_validate(db_feature)
+                    .model_dump()
+                )
+                feature = model_factory(
+                    db_feature.featuretype, db_feature.version, db_feature.appschema
+                ).model_validate(feature_dict | partial_update)
+                session.merge(feature.model_dump_coretable())
+                session.commit()
+                return feature
+            else:
+                raise ValueError(f"no feature found with id {id}")
+
+    def create_tables(self, srid: int) -> None:
+        """Creates coretable and related/spatial tables in the database.
+
+        Args:
+            srid: the EPSG code for spatial data
+        """
+
+        @listens_for(Base.metadata, "before_create")
+        def pre_creation(_, conn, **kwargs):
+            if self.dialect == "sqlite":
+                conn.execute(text("SELECT InitSpatialMetaData('EMPTY')"))
+                conn.execute(text("SELECT InsertEpsgSrid(:srid)"), {"srid": srid})
+
+        @listens_for(Base.metadata, "after_create")
+        def post_creation(_, conn, **kwargs):
+            if self.dialect == "geopackage":
+                conn.execute(
+                    text(
+                        """
+                        INSERT INTO gpkg_extensions (table_name, extension_name, definition, scope)
+                        VALUES
+                            ('gpkg_data_columns', 'gpkg_schema', 'http://www.geopackage.org/spec/#extension_schema', 'read-write'),
+                            ('gpkg_data_column_constraints', 'gpkg_schema', 'http://www.geopackage.org/spec/#extension_schema', 'read-write'),
+                            ('gpkgext_relations', 'related_tables', 'http://www.opengis.net/doc/IS/gpkg-rte/1.0', 'read-write'),
+                            ('refs', 'related_tables', 'http://www.opengis.net/doc/IS/gpkg-rte/1.0', 'read-write')
+                        """
+                    )
+                )
+                conn.execute(
+                    text(
+                        """
+                        INSERT INTO gpkgext_relations (base_table_name, base_primary_column, related_table_name, related_primary_column, relation_name, mapping_table_name)
+                        VALUES
+                            ('coretable', 'id', 'coretable', 'id', 'features', 'refs')
+                        """
+                    )
+                )
+                conn.execute(
+                    text(
+                        """
+                        INSERT INTO gpkg_data_columns (table_name, column_name, mime_type)
+                        VALUES
+                            ('coretable', 'properties', 'application/json')
+                        """
+                    )
+                )
+
+        logger.debug(f"creating tables with srid {srid}")
+        tables = Base.metadata.sorted_tables
+        if not self.dialect == "geopackage":
+            tables.pop(1)
+        tables[0].append_column(
+            Column(
+                "geometry",
+                Geometry(
+                    srid=srid,
+                    spatial_index=True,
+                ),
+                nullable=True,
+            ),
+            replace_existing=True,
+        )
+
+        try:
+            Base.metadata.create_all(self._engine, tables)
+            remove(Base.metadata, "before_create", pre_creation)
+            remove(Base.metadata, "after_create", post_creation)
+
+        except Exception as e:
+            if self.dialect in ["sqlite", "geopackage"]:
+                file = self._engine.url.database
+                Path(file).unlink(missing_ok=True)
+            raise e
+
+    def delete_tables(self) -> None:
+        """Deletes coretable and related/spatial tables from the database."""
+        logger.debug("deleting tables")
+        if self.dialect == "postgresql":
+            command.downgrade(self.alembic_cfg, "base")
+        else:
+            Base.metadata.drop_all(self._engine)

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/xplan_tools/interface/gml.py

@@ -199,9 +199,7 @@ class GMLRepository(BaseRepository):
                     if self.appschema == "xtrasse"
                     else "{http://www.opengis.net/wfs/2.0}member",
                 ).append(
-                    feature.model_dump_gml(
-                        self.appschema, feature_srs=kwargs.get("feature_srs", True)
-                    )
+                    feature.model_dump_gml(feature_srs=kwargs.get("feature_srs", True))
                 )
         bbox = get_envelope(geoms)
         attrib = (

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/xplan_tools/main.py

@@ -22,6 +22,8 @@ __version__ = metadata.version("xplan_tools")
 console = Console()
 error_console = Console(stderr=True, style="bold red")
 logger = logging.getLogger(__name__)
+# don't propagate alembic logs when using CLI
+logging.getLogger("alembic").propagate = False
 
 app = typer.Typer(help=f"XPlan-Tools {__version__}")
 db_app = typer.Typer()

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/xplan_tools/model/adapters/coretable.py

@@ -8,7 +8,9 @@ from xplan_tools.model.orm import Feature, Refs
 class CoretableAdapter:
     """Class to add ORM model - i.e. coretable - transformation methods to XPlan pydantic model via inheritance."""
 
-    def _to_coretable(self) -> Feature:
+    def _to_coretable(
+        self, bulk_mode: bool = False
+    ) -> Feature | tuple[dict, list[dict]]:
         """Converts a BaseFeature to a Coretable Feature object."""
         properties = self.model_dump(mode="json", exclude_none=True)
         id = properties.pop("id")
@@ -95,19 +97,23 @@ class CoretableAdapter:
                     gener_att[f"wert_{item.get_name()}"] = gener_att.pop("wert")
                     gener_att["datatype"] = item.get_name()
                     properties["hatGenerAttribut"].append(gener_att)
-        feature = Feature(
-            id=id,
-            featuretype=self.get_name(),
-            properties=properties,
-            geometry=geometry,
-            appschema=self.get_appschema(),
-            version=self.get_version(),
-        )
-        if refs:
-            feature.refs = [Refs(**ref) for ref in refs]
-        if refs_inv:
-            feature.refs_inv = [Refs(**ref) for ref in refs_inv]
-        return feature
+        feature = {
+            "id": id,
+            "featuretype": self.get_name(),
+            "properties": properties,
+            "geometry": geometry,
+            "appschema": self.get_appschema(),
+            "version": self.get_version(),
+        }
+        if bulk_mode:
+            return feature, [*refs, *refs_inv]
+        else:
+            orm_feature = Feature(**feature)
+            if refs:
+                orm_feature.refs = [Refs(**ref) for ref in refs]
+            if refs_inv:
+                orm_feature.refs_inv = [Refs(**ref) for ref in refs_inv]
+            return orm_feature
 
     @classmethod
     def _from_coretable(cls, feature: Feature) -> dict:

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/xplan_tools/model/adapters/gml.py

@@ -1,7 +1,6 @@
 """Module containing the GMLAdapter for reading from and writing to gml."""
 
 import logging
-from typing import Literal
 from uuid import uuid4
 
 from lxml import etree
@@ -19,7 +18,6 @@ class GMLAdapter:
 
     def _to_etree(
         self,
-        appschema: Literal["xplan", "xtrasse", "plu"] = "xplan",
         **kwargs,
     ) -> etree._Element:
         """Converts XPlan and INSPIRE PLU object to lxml etree Element."""
@@ -62,7 +60,7 @@ class GMLAdapter:
                                         "FORMAT=GML32",
                                         f"GMLID=GML_{uuid4()}",
                                         "SRSNAME_FORMAT=OGC_URL"
-                                        if appschema == "plu"
+                                        if self.get_appschema() == "plu"
                                         else "GML3_LONGSRS=NO",
                                         "NAMESPACE_DECL=YES",
                                     ]
@@ -125,11 +123,11 @@ class GMLAdapter:
                         if isinstance(model_value, list):
                             value_item = model_value[index]
                             etree.SubElement(feature, gml_name).append(
-                                value_item._to_etree(appschema)
+                                value_item._to_etree()
                             )
                         else:
                             etree.SubElement(feature, gml_name).append(
-                                model_value._to_etree(appschema)
+                                model_value._to_etree()
                             )
 
         ns = self.namespace_uri.replace("base/4.0", "base/3.3")

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/xplan_tools/model/base.py

@@ -198,12 +198,14 @@ class BaseCollection(BaseModel):
                             {str(ref): self.features[str(ref)] for ref in refs}
                         )
             yield (
-                plan.name,
-                BaseCollection(
-                    features=collection,
-                    srid=self.srid,
-                    version=self.version,
-                    appschema=self.appschema,
+                (
+                    plan.name,
+                    BaseCollection(
+                        features=collection,
+                        srid=self.srid,
+                        version=self.version,
+                        appschema=self.appschema,
+                    ),
                 )
                 if with_name
                 else BaseCollection(
@@ -211,7 +213,7 @@ class BaseCollection(BaseModel):
                     srid=self.srid,
                     version=self.version,
                     appschema=self.appschema,
-                ),
+                )
             )
 
     # def __iter__(self):
@@ -464,16 +466,21 @@ class BaseFeature(BaseModel, GMLAdapter, CoretableAdapter, JsonFGAdapter):
 
     def model_dump_gml(
         self,
-        appschema: Literal["xplan", "xtrasse", "plu"] = "xplan",
         **kwargs,
     ) -> _Element:
         """Dumps the model data to a GML structure held in an etree.Element."""
-        return self._to_etree(appschema, **kwargs)
+        return self._to_etree(**kwargs)
 
-    def model_dump_coretable(self) -> Feature:
+    def model_dump_coretable(
+        self,
+    ) -> Feature:
         """Dumps the model data to a coretable Feature object to store in a database."""
         return self._to_coretable()
 
+    def model_dump_coretable_bulk(self) -> tuple[dict, list[dict]]:
+        """Dumps the model data to feature and refs dicts to bulk insert in a database."""
+        return self._to_coretable(bulk_mode=True)
+
     def model_dump_jsonfg(
         self,
         **kwargs,

{xplan_tools-1.11.1 → xplan_tools-1.12.0}/xplan_tools/model/migrations/env.py

@@ -22,9 +22,10 @@ def run_migrations_offline() -> None:
     script output.
 
     """
-    url = config.get_main_option("sqlalchemy.url")
     context.configure(
-        url=url,
+        dialect_name=context.get_x_argument(as_dictionary=True).get(
+            "dialect", "postgresql"
+        ),
         target_metadata=target_metadata,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
@@ -41,8 +42,15 @@ def run_migrations_online() -> None:
     and associate a connection with the context.
 
     """
+    cfg_section = config.get_section(config.config_ini_section, {}) or {}
+    x_args = context.get_x_argument(as_dictionary=True)
+    if "sqlalchemy.url" in x_args:
+        cfg_section["sqlalchemy.url"] = x_args["sqlalchemy.url"]
+    elif "url" in x_args:
+        cfg_section["sqlalchemy.url"] = x_args["url"]
+
     connectable = engine_from_config(
-        config.get_section(config.config_ini_section, {}),
+        cfg_section,
         prefix="sqlalchemy.",
         poolclass=pool.NullPool,
     )