threedi-modelchecker 2.6.3.dev1__tar.gz → 2.7.1.dev0__tar.gz

{threedi-modelchecker-2.6.3.dev1 → threedi-modelchecker-2.7.1.dev0}/CHANGES.rst

@@ -2,7 +2,14 @@ Changelog of threedi-modelchecker
 =================================
 
 
-2.6.3 (unreleased)
+2.7.1 (unreleased)
+------------------
+- Add info check 1406 to inform the user if a raster is not compressed.
+- Add check 799 to warn if raster friction pixels are < 1 while Chezy friction is selected
+- Change error message for check 1500
+
+
+2.7.0 (2024-03-12)
 ------------------
 - Support geopackage
 - Support changes in threedi-schema (0.220) needed for geopackage support

{threedi-modelchecker-2.6.3.dev1/threedi_modelchecker.egg-info → threedi-modelchecker-2.7.1.dev0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: threedi-modelchecker
-Version: 2.6.3.dev1
+Version: 2.7.1.dev0
 Summary: Checks validity of a 3Di schematisation
 Author-email: Nelen & Schuurmans <info@nelen-schuurmans.nl>
 License: MIT
@@ -15,7 +15,7 @@ License-File: LICENSE
 Requires-Dist: Click
 Requires-Dist: GeoAlchemy2!=0.11.*,>=0.9
 Requires-Dist: SQLAlchemy>=1.4
-Requires-Dist: threedi-schema==0.220.*
+Requires-Dist: threedi-schema==0.222.dev0
 Provides-Extra: rasterio
 Requires-Dist: rasterio>=1.3; extra == "rasterio"
 Provides-Extra: test

{threedi-modelchecker-2.6.3.dev1 → threedi-modelchecker-2.7.1.dev0}/pyproject.toml

@@ -19,7 +19,7 @@ dependencies = [
     "Click",
     "GeoAlchemy2>=0.9,!=0.11.*",
     "SQLAlchemy>=1.4",
-    "threedi-schema==0.220.*"
+    "threedi-schema==0.222.dev0"
 ]
 
 [project.optional-dependencies]

{threedi-modelchecker-2.6.3.dev1 → threedi-modelchecker-2.7.1.dev0}/threedi_modelchecker/__init__.py

@@ -1,5 +1,5 @@
 from .model_checks import *  # NOQA
 
 # fmt: off
-__version__ = '2.6.3.dev1'
+__version__ = '2.7.1.dev0'
 # fmt: on

{threedi-modelchecker-2.6.3.dev1 → threedi-modelchecker-2.7.1.dev0}/threedi_modelchecker/checks/geo_query.py

@@ -8,7 +8,7 @@ DEFAULT_EPSG = 28992
 
 
 def epsg_code_query():
-    epsg_code = Query(models.GlobalSetting.epsg_code).limit(1).scalar_subquery()
+    epsg_code = Query(models.ModelSettings.epsg_code).limit(1).scalar_subquery()
     return func.coalesce(epsg_code, literal(DEFAULT_EPSG)).label("epsg_code")
 
 

{threedi-modelchecker-2.6.3.dev1 → threedi-modelchecker-2.7.1.dev0}/threedi_modelchecker/checks/other.py

@@ -10,15 +10,6 @@ from .base import BaseCheck, CheckLevel
 from .cross_section_definitions import cross_section_configuration
 from .geo_query import distance, length, transform
 
-# Use these to make checks only work on the first global settings entry:
-first_setting = (
-    Query(models.GlobalSetting.id)
-    .order_by(models.GlobalSetting.id)
-    .limit(1)
-    .scalar_subquery()
-)
-first_setting_filter = models.GlobalSetting.id == first_setting
-
 
 class CorrectAggregationSettingsExist(BaseCheck):
     """Check if aggregation settings are correctly filled with aggregation_method and flow_variable as required"""
@@ -30,30 +21,24 @@ class CorrectAggregationSettingsExist(BaseCheck):
         *args,
         **kwargs,
     ):
-        super().__init__(column=models.GlobalSetting.id, *args, **kwargs)
+        super().__init__(column=models.ModelSettings.id, *args, **kwargs)
         self.aggregation_method = aggregation_method.value
         self.flow_variable = flow_variable.value
 
     def get_invalid(self, session: Session) -> List[NamedTuple]:
-        global_settings = self.to_check(session).filter(first_setting_filter)
+        global_settings = self.to_check(session)
         correctly_defined = session.execute(
-            select(models.AggregationSettings)
-            .filter(
+            select(models.AggregationSettings).filter(
                 models.AggregationSettings.aggregation_method
                 == self.aggregation_method,
                 models.AggregationSettings.flow_variable == self.flow_variable,
             )
-            .filter(
-                models.AggregationSettings.global_settings_id
-                == global_settings.subquery().c.id
-            )
         ).all()
-
         return global_settings.all() if len(correctly_defined) == 0 else []
 
     def description(self) -> str:
         return (
-            "To use the water balance tool, v2_aggregation_settings should have a row where "
+            "To use the water balance tool, aggregation_settings should have a row where "
             f"aggregation_method is {self.aggregation_method} and flow_variable is {self.flow_variable}."
         )
 
@@ -244,12 +229,14 @@ class Use0DFlowCheck(BaseCheck):
     """
 
     def __init__(self, *args, **kwargs):
-        super().__init__(column=models.GlobalSetting.use_0d_inflow, *args, **kwargs)
+        super().__init__(
+            column=models.SimulationTemplateSettings.use_0d_inflow, *args, **kwargs
+        )
 
     def to_check(self, session):
         """Return a Query object on which this check is applied"""
-        return session.query(models.GlobalSetting).filter(
-            models.GlobalSetting.use_0d_inflow != 0
+        return session.query(models.SimulationTemplateSettings).filter(
+            models.SimulationTemplateSettings.use_0d_inflow != 0
         )
 
     def get_invalid(self, session):
@@ -301,7 +288,7 @@ class ConnectionNodes(BaseCheck):
 class ConnectionNodesLength(BaseCheck):
     """Check that the distance between `start_node` and `end_node` is at least
     `min_distance`. The coords will be transformed into (the first entry) of
-    GlobalSettings.epsg_code.
+    ModelSettings.epsg_code.
     """
 
     def __init__(
@@ -469,7 +456,7 @@ class ChannelManholeLevelCheck(BaseCheck):
 
 class OpenChannelsWithNestedNewton(BaseCheck):
     """Checks whether the model has any closed cross-section in use when the
-    NumericalSettings.use_of_nested_newton is turned off.
+    NumericalSettings.use_nested_newton is turned off.
 
     See https://github.com/nens/threeditoolbox/issues/522
     """
@@ -479,7 +466,7 @@ class OpenChannelsWithNestedNewton(BaseCheck):
             column=models.CrossSectionDefinition.id,
             level=level,
             filters=Query(models.NumericalSettings)
-            .filter(models.NumericalSettings.use_of_nested_newton == 0)
+            .filter(models.NumericalSettings.use_nested_newton == 0)
             .exists(),
             *args,
             **kwargs,
@@ -532,8 +519,8 @@ class OpenChannelsWithNestedNewton(BaseCheck):
     def description(self) -> str:
         return (
             f"{self.column_name} has a closed cross section definition while "
-            f"NumericalSettings.use_of_nested_newton is switched off. "
-            f"This gives convergence issues. We recommend setting use_of_nested_newton = 1."
+            f"NumericalSettings.use_nested_newton is switched off. "
+            f"This gives convergence issues. We recommend setting use_nested_newton = 1."
         )
 
 
@@ -731,7 +718,8 @@ class PumpStorageTimestepCheck(BaseCheck):
             .filter(
                 (models.ConnectionNode.storage_area != None)
                 & (
-                    (  # calculate how many seconds the pumpstation takes to empty its storage: (storage * height)/pump capacity
+                    (
+                        # calculate how many seconds the pumpstation takes to empty its storage: (storage * height)/pump capacity
                         (
                             # Arithmetic operations on None return None, so without this
                             # conditional type cast, no invalid results would be returned
@@ -744,9 +732,7 @@ class PumpStorageTimestepCheck(BaseCheck):
                         )
                     )
                     / (models.Pumpstation.capacity / 1000)
-                    < Query(models.GlobalSetting.sim_time_step)
-                    .filter(first_setting_filter)
-                    .scalar_subquery()
+                    < Query(models.TimeStepSettings.time_step).scalar_subquery()
                 )
             )
             .all()
@@ -819,7 +805,7 @@ class InflowNoFeaturesCheck(BaseCheck):
     """Check that the surface table in the global use_0d_inflow setting contains at least 1 feature."""
 
     def __init__(self, *args, surface_table, condition=True, **kwargs):
-        super().__init__(*args, column=models.GlobalSetting.id, **kwargs)
+        super().__init__(*args, column=models.ModelSettings.id, **kwargs)
         self.surface_table = surface_table
         self.condition = condition
 
@@ -828,13 +814,13 @@ class InflowNoFeaturesCheck(BaseCheck):
             select(func.count(self.surface_table.id))
         ).scalar()
         return (
-            session.query(models.GlobalSetting)
+            session.query(models.ModelSettings)
             .filter(self.condition, surface_table_length == 0)
             .all()
         )
 
     def description(self) -> str:
-        return f"v2_global_settings.use_0d_inflow is set to use {self.surface_table.__tablename__}, but {self.surface_table.__tablename__} does not contain any features."
+        return f"model_settings.use_0d_inflow is set to use {self.surface_table.__tablename__}, but {self.surface_table.__tablename__} does not contain any features."
 
 
 class NodeSurfaceConnectionsCheck(BaseCheck):
@@ -1082,3 +1068,48 @@ class AllPresentVariableVegetationParameters(AllPresent):
                 )
             )
         )
+
+
+class UsedSettingsPresentCheck(BaseCheck):
+    def __init__(
+        self,
+        column,
+        settings_table,
+        filters=None,
+        level=CheckLevel.ERROR,
+        error_code=0,
+    ):
+        super().__init__(column, filters, level, error_code)
+        self.settings_table = settings_table
+
+    def get_invalid(self, session: Session) -> List[NamedTuple]:
+        # more than 1 row should be caught by another check
+        all_results = self.to_check(session).filter(self.column == True).all()
+        use_cols = len(all_results) > 0
+        if use_cols and session.query(self.settings_table).count() == 0:
+            return all_results
+        return []
+
+    def description(self) -> str:
+        return f"{self.column_name} in {self.table.name} is set to True but {self.settings_table.__tablename__} is empty"
+
+
+class MaxOneRecordCheck(BaseCheck):
+    def __init__(self, column, filters=None, level=CheckLevel.ERROR, error_code=0):
+        super().__init__(column, filters, level, error_code)
+        self.observed_length = 0
+
+    def get_invalid(self, session: Session) -> List[NamedTuple]:
+        # return mock list in case the table is empty when it shouldn't be
+        all_results = self.to_check(session).all()
+        self.observed_length = len(all_results)
+        if self.observed_length > 1:
+            return all_results if self.observed_length > 0 else ["foo"]
+        else:
+            return []
+
+    def description(self) -> str:
+        return (
+            f"{self.table.name} has {self.observed_length} rows, "
+            f"but should have at most 1 row."
+        )

{threedi-modelchecker-2.6.3.dev1 → threedi-modelchecker-2.7.1.dev0}/threedi_modelchecker/checks/raster.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 from math import isclose
 from pathlib import Path
-from typing import Optional, Set, Type
+from typing import Dict, Optional, Type
 
 from threedi_schema import models
 
@@ -9,6 +9,12 @@ from threedi_modelchecker.interfaces import GDALRasterInterface, RasterInterface
 
 from .base import BaseCheck
 
+# interface_cls.is_valid_geotiff returns False if the dataset is empty
+# (such as when the column doesn't reference a file) or the dataset is not a valid geotiff.
+# It's used on every raster check which reads the raster file in any way,
+# to prevent any other code in the check from executing on a bad file.
+# On such datasets, RasterExistsCheck and/or RasterIsValidCheck will be raised.
+
 
 class Context:
     pass
@@ -16,7 +22,7 @@ class Context:
 
 @dataclass
 class ServerContext(Context):
-    available_rasters: Set[str]
+    available_rasters: Dict[str, str]
     raster_interface: Type[RasterInterface] = GDALRasterInterface
 
 
@@ -62,7 +68,9 @@ class BaseRasterCheck(BaseCheck):
         ]
 
     def get_path_local(self, record, context: LocalContext) -> Optional[str]:
-        abs_path = context.base_path / getattr(record, self.column.name)
+        abs_path = context.base_path.joinpath(
+            "rasters", getattr(record, self.column.name)
+        )
         if abs_path.exists():
             return str(abs_path)
 
@@ -153,7 +161,7 @@ class RasterHasMatchingEPSGCheck(BaseRasterCheck):
     """Check whether a raster's EPSG code matches the EPSG code in the global settings for the SQLite."""
 
     def get_invalid(self, session):
-        epsg_code_query = session.query(models.GlobalSetting.epsg_code).first()
+        epsg_code_query = session.query(models.ModelSettings.epsg_code).first()
         if epsg_code_query is not None:
             self.epsg_code = epsg_code_query[0]
         else:
@@ -169,7 +177,7 @@ class RasterHasMatchingEPSGCheck(BaseRasterCheck):
             return raster.epsg_code == self.epsg_code
 
     def description(self):
-        return f"The file in {self.column_name} has no EPSG code or the EPSG code does not match does not match v2_global_settings.epsg_code"
+        return f"The file in {self.column_name} has no EPSG code or the EPSG code does not match does not match model_settings.epsg_code"
 
 
 class RasterSquareCellsCheck(BaseRasterCheck):
@@ -196,9 +204,11 @@ class RasterGridSizeCheck(BaseRasterCheck):
     """Check whether the global settings' grid size is an even multiple of a raster's cell size (at least 2x)."""
 
     def get_invalid(self, session):
-        grid_space_query = session.query(models.GlobalSetting.grid_space).first()
-        if grid_space_query is not None:
-            self.grid_space = grid_space_query[0]
+        minimum_cell_size_query = session.query(
+            models.ModelSettings.minimum_cell_size
+        ).first()
+        if minimum_cell_size_query is not None:
+            self.minimum_cell_size = minimum_cell_size_query[0]
         else:
             return []
         return super().get_invalid(session)
@@ -211,17 +221,17 @@ class RasterGridSizeCheck(BaseRasterCheck):
             try:
                 return (
                     isclose(
-                        a=((self.grid_space / raster.pixel_size[0]) % 2),
+                        a=((self.minimum_cell_size / raster.pixel_size[0]) % 2),
                         b=0,
                         rel_tol=1e-09,
                     )
-                ) and (self.grid_space >= (2 * raster.pixel_size[0]))
+                ) and (self.minimum_cell_size >= (2 * raster.pixel_size[0]))
             # if one of the fields is a NoneType it will be caught elsewhere
             except TypeError:
                 return True
 
     def description(self):
-        return "v2_global_settings.grid_space is not a positive even multiple of the raster cell size."
+        return "model_settings.minimum_cell_size is not a positive even multiple of the raster cell size."
 
 
 class RasterPixelCountCheck(BaseRasterCheck):
@@ -300,6 +310,19 @@ class RasterRangeCheck(BaseRasterCheck):
         return f"{self.column_name} has values {' and/or '.join(parts)} or is empty"
 
 
+class RasterCompressionUsedCheck(BaseRasterCheck):
+    """Checks whether compression was used for the raster"""
+
+    def is_valid(self, path: str, interface_cls: Type[RasterInterface]):
+        with interface_cls(path) as raster:
+            if not raster.is_valid_geotiff:
+                return True
+            return raster.compression != "NONE"
+
+    def description(self):
+        return f"Raster {self.column_name} is not compressed. It is recommended to use DEFLATE compression. This speeds up uploading and downloading and reduces storage space."
+
+
 @dataclass
 class GDALUnavailable:
     id: int