xplan-tools 1.11.1__py3-none-any.whl → 1.12.1__py3-none-any.whl

xplan_tools/interface/db.py

@@ -6,10 +6,20 @@ from pathlib import Path
 from typing import Iterable
 
 from alembic import command, config, script
-from alembic.runtime import migration
 from geoalchemy2 import load_spatialite_gpkg
 from geoalchemy2.admin.dialects.sqlite import load_spatialite_driver
-from sqlalchemy import DDL, Column, Engine, create_engine, inspect, text
+from sqlalchemy import (
+    Column,
+    Engine,
+    MetaData,
+    Table,
+    create_engine,
+    delete,
+    insert,
+    inspect,
+    select,
+    text,
+)
 from sqlalchemy.engine import URL, make_url
 
 # from sqlalchemy.dialects.sqlite.base import SQLiteCompiler
@@ -21,7 +31,7 @@ from sqlalchemy.orm import sessionmaker
 # from sqlalchemy.sql.expression import BindParameter
 from xplan_tools.model import model_factory
 from xplan_tools.model.base import BaseCollection, BaseFeature
-from xplan_tools.model.orm import Base, Feature, Geometry
+from xplan_tools.model.orm import Base, Feature, Geometry, Refs
 from xplan_tools.util import check_schema_accessibility
 
 # from xplan_tools.util import linearize_geom
@@ -62,147 +72,124 @@ class DBRepository(BaseRepository):
         self.srid = srid
         self.with_views = with_views
 
-        alembic_cfg = config.Config()
-        alembic_cfg.set_main_option("script_location", "xplan_tools:model:migrations")
-        alembic_cfg.set_main_option(
+        self.alembic_cfg = config.Config()
+        self.alembic_cfg.set_main_option(
+            "script_location", "xplan_tools:model:migrations"
+        )
+        self.alembic_cfg.set_main_option("srid", str(srid))
+        if with_views:
+            self.alembic_cfg.set_main_option("with_views", "1")
+        self.alembic_cfg.set_main_option(
             "sqlalchemy.url",
             datasource.replace("gpkg:", "sqlite:").replace(
                 "postgresql:", "postgresql+psycopg:"
             ),
         )
-        alembic_url = make_url(alembic_cfg.get_main_option("sqlalchemy.url"))
         if self.schema and self.dialect == "postgresql":
-            alembic_cfg.set_main_option("custom_schema", self.schema)
-        current_version = script.ScriptDirectory.from_config(alembic_cfg).get_heads()
-        alembic_engine = create_engine(alembic_url)
+            check_schema_accessibility(self._engine, self.schema)
+            self.alembic_cfg.set_main_option("custom_schema", self.schema)
+        current_version = script.ScriptDirectory.from_config(
+            self.alembic_cfg
+        ).get_heads()
         # test for tables and revision
-        with alembic_engine.connect() as conn:
-            context = migration.MigrationContext.configure(
-                conn,
-            )
-            db_version = context.get_current_heads()
+        with self._engine.connect() as conn:
             inspector = inspect(conn)
             tables = inspector.get_table_names(schema=self.schema)
-        is_coretable = {"coretable", "refs"}.issubset(set(tables))
-        is_current_version = set(db_version) == set(current_version)
-
-        # handle schema upgrade or table creation
-        if db_version:
-            if not is_current_version:
-                if self.dialect == "postgresql":
-                    logger.info("Running database migrations")
-                    command.upgrade(alembic_cfg, "head")
-                else:
-                    e = RuntimeError(
-                        f"Incompatible database revision and automatic migration not implemented for {self.dialect}"
-                    )
-                    e.add_note(
-                        "please set up a new database with the current version of this library"
-                    )
-                    raise e
+            is_coretable = {"coretable", "refs"}.issubset(set(tables))
+            if "alembic_version" in tables:
+                alembic_table = Table(
+                    "alembic_version",
+                    MetaData(schema=self.schema),
+                    Column("version_num"),
+                )
+                stmt = select(alembic_table.c.version_num)
+                db_version = conn.execute(stmt).scalars().all()
             else:
+                db_version = []
+            is_current_version = set(db_version) == set(current_version)
+            if is_current_version:
                 logger.info("Database is at current revision")
+                return
+        # handle schema upgrade or table creation
+        if is_coretable and not db_version:
+            e = RuntimeError("Coretable with no revision found in database")
+            e.add_note(
+                "it is likely that the database was set up with an older version of this library which didn't use revisions yet"
+            )
+            e.add_note(
+                "please set up a new database or add a revision corresponding to the current model manually"
+            )
+            raise e
+        # if postgresql, run alembic and return
+        elif self.dialect == "postgresql":
+            logger.info(
+                "Running database migrations"
+                if db_version
+                else "Creating new database schema"
+            )
+            command.upgrade(self.alembic_cfg, "head")
+            return
+        elif db_version:
+            e = NotImplementedError(
+                f"Incompatible database revision and automatic migration not implemented for {self.dialect}"
+            )
+            e.add_note(
+                "please set up a new database with the current version of this library"
+            )
+            raise e
         else:
-            if is_coretable:
-                e = RuntimeError("Coretable with no revision found in database")
-                e.add_note(
-                    "it is likely that the database was set up with an older version of this library which didn't use revisions yet"
-                )
-                e.add_note(
-                    "please set up a new database or add a revision corresponding to the current model manually"
-                )
-                raise e
-            else:
-                # create tables if it's a fresh DB and set it to current revision
-                logger.info("Creating new database schema")
-                self.create_tables(self.srid, self.with_views)
-                command.stamp(alembic_cfg, "head")
+            # create tables if it's a fresh file-based DB and set it to current revision
+            logger.info("Creating new database schema")
+            self.create_tables(self.srid)
+            command.stamp(self.alembic_cfg, "head")
 
     @property
     def _engine(self) -> Engine:
+        url = (
+            self.datasource.set(drivername="postgresql+psycopg")
+            if self.dialect == "postgresql"
+            else self.datasource
+        )
+        connect_args: dict[str, str] = {}
+        if self.schema and self.dialect == "postgresql":
+            connect_args["options"] = f"-csearch_path={self.schema},public"
+        engine = create_engine(url, connect_args=connect_args)
         if self.dialect == "geopackage":
-            engine = create_engine(self.datasource)
             listen(engine, "connect", load_spatialite_gpkg)
-            return engine
         elif self.dialect == "sqlite":
-            engine = create_engine(self.datasource)
             listen(
                 engine,
                 "connect",
                 load_spatialite_driver,
             )
-            return engine
-        else:
-            engine = create_engine(
-                self.datasource.set(drivername="postgresql+psycopg"), echo=False
-            )
-            if self.schema:
-                check_schema_accessibility(engine, self.schema)
-
-            return engine
-
-    # see https://docs.sqlalchemy.org/en/20/faq/sqlexpressions.html#rendering-bound-parameters-inline
-    # @compiles(BindParameter)
-    # def _render_literal_bindparam(
-    #     element: BindParameter, compiler, dump_to_file=False, **kw
-    # ):
-    #     if not dump_to_file:
-    #         return compiler.visit_bindparam(element, **kw)
-    #     if (
-    #         isinstance(compiler, SQLiteCompiler)
-    #         and "geometry" in str(element.type)
-    #         and element.value is not None
-    #     ):
-    #         return repr(linearize_geom(element.value))
-    #     elif isinstance(element.value, dict):
-    #         return repr(json.dumps(element.value))
-    #     else:
-    #         return repr(str(element.value))
+        return engine
 
     def get_plan_by_id(self, id: str) -> BaseCollection:
         logger.debug(f"retrieving plan with id {id}")
         with self.Session() as session:
-            stmt = text(
-                "SELECT srs_id FROM gpkg_geometry_columns WHERE table_name='coretable'"
-                if self.dialect == "geopackage"
-                else "SELECT srid FROM geometry_columns WHERE f_table_name='coretable'"
-            )
-            srid = session.execute(stmt).scalar_one()
-
-            feature = session.get(Feature, id)
-            if not feature:
+            plan_feature = session.get(Feature, id)
+            if not plan_feature:
                 raise ValueError(f"no feature found with id {id}")
-            elif "Plan" not in feature.featuretype:
-                raise ValueError(f"{feature.featuretype} is not a plan object")
+            elif "Plan" not in plan_feature.featuretype:
+                raise ValueError(f"{plan_feature.featuretype} is not a plan object")
             else:
-                self.version = (
-                    "2.0" if feature.appschema == "xtrasse" else feature.version
-                )
-                collection = {
-                    id: model_factory(
-                        feature.featuretype, self.version, feature.appschema
+                plan_model = model_factory(
+                    plan_feature.featuretype,
+                    plan_feature.version,
+                    plan_feature.appschema,
+                ).model_validate(plan_feature)
+                collection = {id: plan_model}
+                srid = plan_model.get_geom_srid()
+                # iterate related features with depth=2: plan -> section -> features
+                for feature in plan_feature.related_features(session, depth=2):
+                    collection[str(feature.id)] = model_factory(
+                        feature.featuretype, feature.version, feature.appschema
                     ).model_validate(feature)
-                }
-                for ref in feature.refs:
-                    collection[ref.feature_inv.id] = model_factory(
-                        ref.feature_inv.featuretype, self.version, feature.appschema
-                    ).model_validate(ref.feature_inv)
-                    if ref.rel == "bereich":
-                        for obj in ref.feature_inv.refs:
-                            collection[obj.feature_inv.id] = model_factory(
-                                obj.feature_inv.featuretype,
-                                self.version,
-                                feature.appschema,
-                            ).model_validate(obj.feature_inv)
-                for ref_inv in feature.refs_inv:
-                    collection[ref_inv.feature.id] = model_factory(
-                        ref_inv.feature.featuretype, self.version, feature.appschema
-                    ).model_validate(ref_inv.feature)
                 return BaseCollection(
                     features=collection,
                     srid=srid,
-                    version=feature.version,
-                    appschema=feature.appschema,
+                    version=plan_feature.version,
+                    appschema=plan_feature.appschema,
                 )
 
     def get(self, id: str) -> BaseFeature:
@@ -228,24 +215,25 @@ class DBRepository(BaseRepository):
     def delete_plan_by_id(self, id: str) -> BaseFeature:
         logger.debug(f"deleting plan with id {id}")
         with self.Session() as session:
-            feature = session.get(Feature, id)
-            if not feature:
+            plan_feature = session.get(Feature, id)
+            if not plan_feature:
                 raise ValueError(f"no feature found with id {id}")
-            elif "Plan" not in feature.featuretype:
-                raise ValueError(f"{feature.featuretype} is not a plan object")
+            elif "Plan" not in plan_feature.featuretype:
+                raise ValueError(f"{plan_feature.featuretype} is not a plan object")
             else:
-                for ref in feature.refs:
-                    if ref.rel == "bereich":
-                        for bereich_ref in ref.feature_inv.refs:
-                            session.delete(bereich_ref.feature_inv)
-                    session.delete(ref.feature_inv)
-                for ref_inv in feature.refs_inv:
-                    session.delete(ref_inv.feature)
-                session.delete(feature)
+                plan_model = model_factory(
+                    plan_feature.featuretype,
+                    plan_feature.version,
+                    plan_feature.appschema,
+                ).model_validate(plan_feature)
+                ids = [plan_feature.id]
+                ids += [
+                    feature.id for feature in plan_feature.related_features(session)
+                ]
+                stmt = delete(Feature).where(Feature.id.in_(ids))
+                session.execute(stmt)
                 session.commit()
-                return model_factory(
-                    feature.featuretype, feature.version, feature.appschema
-                ).model_validate(feature)
+                return plan_model
 
     def delete(self, id: str) -> BaseFeature:
         logger.debug(f"deleting feature with id {id}")
@@ -265,15 +253,20 @@ class DBRepository(BaseRepository):
     ) -> None:
         logger.debug("saving collection")
         with self.Session() as session:
+            feature_list = []
+            refs_list = []
             for feature in (
                 features.get_features()
                 if isinstance(features, BaseCollection)
                 else features
             ):
-                feature = feature.model_dump_coretable()
-                if session.get(Feature, feature.id):
-                    raise ValueError(f"feature with id {feature.id} already exists")
-                session.merge(feature)
+                feature, refs = feature.model_dump_coretable_bulk()
+                feature_list.append(feature)
+                refs_list.extend([ref for ref in refs if ref not in refs_list])
+            if feature_list:
+                session.execute(insert(Feature), feature_list)
+            if refs_list:
+                session.execute(insert(Refs), refs_list)
             session.commit()
 
     def update_all(
@@ -322,32 +315,21 @@ class DBRepository(BaseRepository):
             else:
                 raise ValueError(f"no feature found with id {id}")
 
-    def create_tables(self, srid: int, with_views: bool = False) -> None:
+    def create_tables(self, srid: int) -> None:
         """Creates coretable and related/spatial tables in the database.
 
         Args:
             srid: the EPSG code for spatial data
-            with_views: whether to create geometrytype-specific views (postgres only)
         """
 
         @listens_for(Base.metadata, "before_create")
-        def pre_creation(metadata, conn, **kw):
+        def pre_creation(_, conn, **kwargs):
             if self.dialect == "sqlite":
                 conn.execute(text("SELECT InitSpatialMetaData('EMPTY')"))
                 conn.execute(text("SELECT InsertEpsgSrid(:srid)"), {"srid": srid})
 
         @listens_for(Base.metadata, "after_create")
-        def post_creation(metadata, conn, **kw):
-            coretable = self.schema + ".coretable" if self.schema else "coretable"
-            stmt = (
-                DDL(
-                    "CREATE INDEX IF NOT EXISTS idx_coretable_geometry ON %(coretable)s USING GIST (geometry)",
-                    {"coretable": coretable},
-                )
-                if self.dialect == "postgresql"
-                else text("SELECT CreateSpatialIndex(:coretable, 'geometry')")
-            )
-            conn.execute(stmt, {"coretable": coretable})
+        def post_creation(_, conn, **kwargs):
             if self.dialect == "geopackage":
                 conn.execute(
                     text(
@@ -362,83 +344,40 @@ class DBRepository(BaseRepository):
                     )
                 )
                 conn.execute(
-                    text("""
-                            INSERT INTO gpkgext_relations (base_table_name, base_primary_column, related_table_name, related_primary_column, relation_name, mapping_table_name)
-                            VALUES
-                                ('coretable', 'id', 'coretable', 'id', 'features', 'refs')
-                            """)
+                    text(
+                        """
+                        INSERT INTO gpkgext_relations (base_table_name, base_primary_column, related_table_name, related_primary_column, relation_name, mapping_table_name)
+                        VALUES
+                            ('coretable', 'id', 'coretable', 'id', 'features', 'refs')
+                        """
+                    )
                 )
                 conn.execute(
-                    text("""
-                            INSERT INTO gpkg_data_columns (table_name, column_name, mime_type)
-                            VALUES
-                                ('coretable', 'properties', 'application/json')
-                            """)
-                )
-            if with_views:
-                if self.dialect != "postgresql":
-                    logger.warning(
-                        f"Creating views not yet supported for {self.dialect}, skipping"
-                    )
-                else:
-                    conn.execute(
-                        DDL(
-                            """
-                                create or replace view %(schema)s.coretable_points as
-                                select pk, id, featuretype, properties, ST_Multi(geometry)::geometry(MultiPoint, %(srid)s) as geometry, appschema, version
-                                from %(schema)s.coretable
-                                where geometry_type = 'point'
-                                """,
-                            {"srid": srid, "schema": self.schema or "public"},
-                        )
-                    )
-                    conn.execute(
-                        DDL(
-                            """
-                                create or replace view %(schema)s.coretable_lines as
-                                select pk, id, featuretype, properties, ST_Multi(ST_ForceCurve(geometry))::geometry(MultiCurve, %(srid)s) as geometry, appschema, version
-                                from %(schema)s.coretable
-                                where geometry_type = 'line'
-                                """,
-                            {"srid": srid, "schema": self.schema or "public"},
-                        )
-                    )
-                    conn.execute(
-                        DDL(
-                            """
-                                create or replace view %(schema)s.coretable_polygons as
-                                select pk, id, featuretype, properties, ST_Multi(ST_ForceCurve(geometry))::geometry(MultiSurface, %(srid)s) as geometry, appschema, version
-                                from %(schema)s.coretable
-                                where geometry_type = 'polygon'
-                                """,
-                            {"srid": srid, "schema": self.schema or "public"},
-                        )
-                    )
-                    conn.execute(
-                        DDL(
-                            """
-                                create or replace view %(schema)s.coretable_nogeoms as
-                                select pk, id, featuretype, properties, geometry, appschema, version
-                                from %(schema)s.coretable
-                                where geometry_type = 'nogeom'
-                                """,
-                            {"schema": self.schema or "public"},
-                        ),
+                    text(
+                        """
+                        INSERT INTO gpkg_data_columns (table_name, column_name, mime_type)
+                        VALUES
+                            ('coretable', 'properties', 'application/json')
+                        """
                     )
+                )
 
         logger.debug(f"creating tables with srid {srid}")
         tables = Base.metadata.sorted_tables
         if not self.dialect == "geopackage":
             tables.pop(1)
         tables[0].append_column(
-            Column("geometry", Geometry(srid=srid, spatial_index=False), nullable=True),
+            Column(
+                "geometry",
+                Geometry(
+                    srid=srid,
+                    spatial_index=True,
+                ),
+                nullable=True,
+            ),
             replace_existing=True,
         )
 
-        if self.schema and self.dialect.startswith("postgresql"):
-            for table in Base.metadata.tables.values():
-                table.schema = self.schema
-
         try:
             Base.metadata.create_all(self._engine, tables)
             remove(Base.metadata, "before_create", pre_creation)
@@ -453,7 +392,7 @@ class DBRepository(BaseRepository):
     def delete_tables(self) -> None:
         """Deletes coretable and related/spatial tables from the database."""
         logger.debug("deleting tables")
-        if self.schema and self.dialect.startswith("postgresql"):
-            for table in Base.metadata.tables.values():
-                table.schema = self.schema
-        Base.metadata.drop_all(self._engine)
+        if self.dialect == "postgresql":
+            command.downgrade(self.alembic_cfg, "base")
+        else:
+            Base.metadata.drop_all(self._engine)

xplan_tools/interface/gml.py

@@ -199,9 +199,7 @@ class GMLRepository(BaseRepository):
                     if self.appschema == "xtrasse"
                     else "{http://www.opengis.net/wfs/2.0}member",
                 ).append(
-                    feature.model_dump_gml(
-                        self.appschema, feature_srs=kwargs.get("feature_srs", True)
-                    )
+                    feature.model_dump_gml(feature_srs=kwargs.get("feature_srs", True))
                 )
         bbox = get_envelope(geoms)
         attrib = (

xplan_tools/main.py

@@ -22,6 +22,8 @@ __version__ = metadata.version("xplan_tools")
 console = Console()
 error_console = Console(stderr=True, style="bold red")
 logger = logging.getLogger(__name__)
+# don't propagate alembic logs when using CLI
+logging.getLogger("alembic").propagate = False
 
 app = typer.Typer(help=f"XPlan-Tools {__version__}")
 db_app = typer.Typer()

xplan_tools/model/adapters/coretable.py

@@ -8,7 +8,9 @@ from xplan_tools.model.orm import Feature, Refs
 class CoretableAdapter:
     """Class to add ORM model - i.e. coretable - transformation methods to XPlan pydantic model via inheritance."""
 
-    def _to_coretable(self) -> Feature:
+    def _to_coretable(
+        self, bulk_mode: bool = False
+    ) -> Feature | tuple[dict, list[dict]]:
         """Converts a BaseFeature to a Coretable Feature object."""
         properties = self.model_dump(mode="json", exclude_none=True)
         id = properties.pop("id")
@@ -95,19 +97,23 @@ class CoretableAdapter:
                     gener_att[f"wert_{item.get_name()}"] = gener_att.pop("wert")
                     gener_att["datatype"] = item.get_name()
                     properties["hatGenerAttribut"].append(gener_att)
-        feature = Feature(
-            id=id,
-            featuretype=self.get_name(),
-            properties=properties,
-            geometry=geometry,
-            appschema=self.get_appschema(),
-            version=self.get_version(),
-        )
-        if refs:
-            feature.refs = [Refs(**ref) for ref in refs]
-        if refs_inv:
-            feature.refs_inv = [Refs(**ref) for ref in refs_inv]
-        return feature
+        feature = {
+            "id": id,
+            "featuretype": self.get_name(),
+            "properties": properties,
+            "geometry": geometry,
+            "appschema": self.get_appschema(),
+            "version": self.get_version(),
+        }
+        if bulk_mode:
+            return feature, [*refs, *refs_inv]
+        else:
+            orm_feature = Feature(**feature)
+            if refs:
+                orm_feature.refs = [Refs(**ref) for ref in refs]
+            if refs_inv:
+                orm_feature.refs_inv = [Refs(**ref) for ref in refs_inv]
+            return orm_feature
 
     @classmethod
     def _from_coretable(cls, feature: Feature) -> dict:

xplan_tools/model/adapters/gml.py

@@ -1,7 +1,6 @@
 """Module containing the GMLAdapter for reading from and writing to gml."""
 
 import logging
-from typing import Literal
 from uuid import uuid4
 
 from lxml import etree
@@ -19,7 +18,6 @@ class GMLAdapter:
 
     def _to_etree(
         self,
-        appschema: Literal["xplan", "xtrasse", "plu"] = "xplan",
         **kwargs,
     ) -> etree._Element:
         """Converts XPlan and INSPIRE PLU object to lxml etree Element."""
@@ -62,7 +60,7 @@ class GMLAdapter:
                                         "FORMAT=GML32",
                                         f"GMLID=GML_{uuid4()}",
                                         "SRSNAME_FORMAT=OGC_URL"
-                                        if appschema == "plu"
+                                        if self.get_appschema() == "plu"
                                         else "GML3_LONGSRS=NO",
                                         "NAMESPACE_DECL=YES",
                                     ]
@@ -125,11 +123,11 @@ class GMLAdapter:
                         if isinstance(model_value, list):
                             value_item = model_value[index]
                             etree.SubElement(feature, gml_name).append(
-                                value_item._to_etree(appschema)
+                                value_item._to_etree()
                             )
                         else:
                             etree.SubElement(feature, gml_name).append(
-                                model_value._to_etree(appschema)
+                                model_value._to_etree()
                             )
 
         ns = self.namespace_uri.replace("base/4.0", "base/3.3")