hydamo-validation 1.3.0b3__tar.gz → 1.3.0b5__tar.gz

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydamo_validation
-Version: 1.3.0b3
+Version: 1.3.0b5
 Summary: Validation module for HyDAMO data
 Author-email: Daniel Tollenaar <daniel@d2hydro.nl>
 License: MIT

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b5}/hydamo_validation/__init__.py

@@ -1,13 +1,14 @@
 __author__ = ["Het Waterschapshuis", "D2HYDRO", "HKV", "HydroConsult"]
 __copyright__ = "Copyright 2021, HyDAMO ValidatieTool"
 __credits__ = ["D2HYDRO", "HKV", "HydroConsult"]
-__version__ = "1.3.0b3"
+__version__ = "1.3.0b5"
 
 __license__ = "MIT"
 __maintainer__ = "Daniel Tollenaar"
 __email__ = "daniel@d2hydro.nl"
 
-import fiona
+import fiona  # top-level import to avoid fiona import issue: https://github.com/conda-forge/fiona-feedstock/issues/213
+from pathlib import Path
 from hydamo_validation.functions import topologic as topologic_functions
 from hydamo_validation.functions import logic as logic_functions
 from hydamo_validation.functions import general as general_functions
@@ -19,4 +20,5 @@ __all__ = [
     "logic_functions",
     "general_functions",
     "validator",
+    "SCHEMAS_PATH",
 ]

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b5}/hydamo_validation/logical_validation.py

@@ -1,5 +1,6 @@
 """Logical validation."""
 
+# %%
 from hydamo_validation import general_functions, logic_functions, topologic_functions
 from shapely.geometry import LineString, Point, Polygon
 import numpy as np
@@ -102,9 +103,9 @@ def gdf_add_summary(
     if critical:
         gdf.loc[gdf[variable] == False, "invalid_critical"] += f"{rule_id}{separator}"
     else:
-        gdf.loc[
-            gdf[variable] == False, "invalid_non_critical"
-        ] += f"{rule_id}{separator}"
+        gdf.loc[gdf[variable] == False, "invalid_non_critical"] += (
+            f"{rule_id}{separator}"
+        )
     if tags is not None:
         gdf.loc[tags_indices, ("tags_assigned")] += f"{tags}{separator}"
         gdf.loc[gdf[variable] == False, "tags_invalid"] += f"{tags}{separator}"
@@ -122,16 +123,19 @@ def execute(
 ):
     """Execute the logical validation."""
 
-    object_rules_sets = (
+    object_rules_sets = [
         i
         for i in validation_rules_sets["objects"]
         if i["object"] in datamodel.data_layers
+    ]
+    logger.info(
+        rf"lagen met valide objecten en regels: {[i["object"] for i in object_rules_sets]}"
     )
     for object_rules in object_rules_sets:
         col_translation: dict = {}
 
         object_layer = object_rules["object"]
-        logger.info(f"start logical validations of layer: {object_layer}")
+        logger.info(f"{object_layer}: start")
         object_gdf = getattr(datamodel, object_layer).copy()
 
         # add summary columns
@@ -145,7 +149,7 @@ def execute(
             general_rules_sorted = sorted(general_rules, key=lambda k: k["id"])
             for rule in general_rules_sorted:
                 logger.info(
-                    f"{object_layer}: executing general-rule with id {rule['id']}"
+                    f"{object_layer}: uitvoeren general-rule met id {rule['id']}"
                 )
                 try:
                     result_variable = rule["result_variable"]
@@ -224,7 +228,7 @@ def execute(
             try:
                 rule_id = rule["id"]
                 logger.info(
-                    f"{object_layer}: executing validation-rule with id {rule_id} ({rule['name']})"
+                    f"{object_layer}: uitvoeren validatieregel met id {rule_id} ({rule['name']})"
                 )
                 result_variable = rule["result_variable"]
                 if "exceptions" in rule.keys():

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b5}/hydamo_validation/summaries.py

@@ -142,7 +142,7 @@ class LayersSummary:
                         gdf_out = gdf.copy()
                         if gdf_out.crs:
                             gdf_out.to_crs("epsg:4326", inplace=True)
-                        gdf_out.to_file(file_path, driver="GeoJSON")
+                        gdf_out.to_file(file_path, driver="GeoJSON", engine="pyogrio")
 
                     # drop geometry for writing to csv
                     elif output_type == "csv":
@@ -150,7 +150,7 @@ class LayersSummary:
                             output_type, f"{object_layer}.csv"
                         )
                         df = gdf.drop("geometry", axis=1)
-                        df.to_csv(file_path)
+                        df.to_csv(file_path, index=False)
 
                     # write to geopackage as is
                     elif output_type == "geopackage":
@@ -161,6 +161,7 @@ class LayersSummary:
                             layer=object_layer,
                             driver="GPKG",
                             engine="pyogrio",
+                            layer_options={"OVERWRITE": "YES"},
                         )
                 layers += [object_layer]
             else:
@@ -182,7 +183,7 @@ class ResultSummary:
         self.error_layers = []
         self.syntax_result = []
         self.validation_result = []
-        self.error = None
+        self.error = []
         self.errors = None
         self.warnings = None
 

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b5}/hydamo_validation/validator.py

@@ -23,7 +23,7 @@ import traceback
 
 OUTPUT_TYPES = ["geopackage", "geojson", "csv"]
 LOG_LEVELS = Literal["INFO", "DEBUG"]
-INCLUDE_COLUMNS = ["code"]
+INCLUDE_COLUMNS = ["nen3610id", "code"]
 SCHEMAS_PATH = Path(__file__).parent.joinpath(r"./schemas")
 HYDAMO_SCHEMAS_PATH = SCHEMAS_PATH.joinpath("hydamo")
 RULES_SCHEMAS_PATH = SCHEMAS_PATH.joinpath("rules")
@@ -54,6 +54,13 @@ def _add_log_file(logger, log_file):
     return logger
 
 
+def _close_log_file(logger):
+    """Remove log-file from existing logger"""
+    for h in logger.handlers:
+        h.close()
+        logger.removeHandler(h)
+
+
 def _log_to_results(log_file, result_summary):
     result_summary.log = log_file.read_text().split("\n")
 
@@ -180,28 +187,31 @@ def _validator(
             log_level=log_level,
         )
 
-        logger.info("validator start")
+        logger.info("init validatie")
         date_check = pd.Timestamp.now().isoformat()
         result_summary = ResultSummary(date_check=date_check)
         layers_summary = LayersSummary(date_check=date_check)
         # check if all files are present
         # create a results_path
+        permission_error = False
         if dir_path.exists():
             results_path = dir_path.joinpath("results")
             if results_path.exists():
                 try:
                     shutil.rmtree(results_path)
                 except PermissionError:
-                    logger.warning(
-                        f"Cannot remove {results_path}. This may lead to write-issues later!"
-                    )
-                    pass
+                    permission_error = True
             results_path.mkdir(parents=True, exist_ok=True)
         else:
             raise FileNotFoundError(f"{dir_path.absolute().resolve()} does not exist")
 
         log_file = results_path.joinpath("validator.log")
         logger = _add_log_file(logger, log_file=log_file)
+        logger.info("start validatie")
+        if permission_error:
+            logger.warning(
+                f"Kan pad {results_path} niet verwijderen. Dit kan later tot problemen leiden!"
+            )
         dataset_path = dir_path.joinpath("datasets")
         validation_rules_json = dir_path.joinpath("validationrules.json")
         missing_paths = []
@@ -209,11 +219,11 @@ def _validator(
             if not path.exists():
                 missing_paths += [str(path)]
         if missing_paths:
-            result_summary.error = [f'missing_paths: {",".join(missing_paths)}']
+            result_summary.error += [f'missing_paths: {",".join(missing_paths)}']
             raise FileNotFoundError(f'missing_paths: {",".join(missing_paths)}')
         else:
             validation_rules_sets = read_validation_rules(
-                validation_rules_json, SCHEMAS_PATH
+                validation_rules_json, result_summary
             )
 
         # check if output-files are supported
@@ -224,7 +234,7 @@ def _validator(
             error_message = (
                 r"unsupported output types: " f'{",".join(unsupported_output_types)}'
             )
-            result_summary.error = [error_message]
+            result_summary.error += [error_message]
             raise TypeError(error_message)
 
         # set coverages
@@ -249,7 +259,7 @@ def _validator(
         result_summary.dataset_layers = datasets.layers
 
         ## validate syntax of datasets on layers-level and append to result
-        logger.info("start syntax-validation of object-layers")
+        logger.info("start syntax-validatie van object-lagen")
         valid_layers = datamodel_layers(datamodel.layers, datasets.layers)
         result_summary.missing_layers = missing_layers(
             datamodel.layers, datasets.layers
@@ -265,17 +275,15 @@ def _validator(
             status_object = validation_rules_sets["status_object"]
 
         for layer in valid_layers:
-            logger.info(f"read layer {layer}")
+            logger.info(f"{layer}: inlezen")
 
             # read layer
             gdf, schema = datasets.read_layer(
                 layer, result_summary=result_summary, status_object=status_object
             )
-            if (
-                gdf.empty
-            ):  # pass if gdf is empty. Most likely due to mall-formed or ill-specifiec status_object
+            if gdf.empty:  # pass if gdf is empty. Most likely due to mall-formed or ill-specifiec status_object
                 logger.warning(
-                    f"layer {layer} is empty. Be aware only values of {status_object} in field 'statusobject' are read!"
+                    f"{layer}: geen objecten ingelezen. Zorg dat alle waarden in de kolom 'status_object' voorkomen in {status_object}"
                 )
                 continue
 
@@ -285,23 +293,28 @@ def _validator(
                     gdf[col] = None
                     schema["properties"][col] = "str"
 
-            logger.info(f"syntax-validation of fields in layer {layer}")
+            logger.info(f"{layer}: syntax-validatie")
             gdf, result_gdf = fields_syntax(
                 gdf,
-                schema,
-                datamodel.validation_schemas[layer],
+                schema=schema,
+                validation_schema=datamodel.validation_schemas[layer],
                 keep_columns=INCLUDE_COLUMNS,
             )
 
             # Add the syntax-validation result to the results_summary
             layers_summary.set_data(result_gdf, layer, schema["geometry"])
             # Add the corrected datasets_layer data to the datamodel.
-            datamodel.set_data(gdf, layer, index_col=None)
+            if gdf.empty:
+                logger.warning(
+                    f"{layer}: geen valide objecten na syntax-validatie. Inspecteer 'syntax_oordeel' in de resultaten; deze is false voor alle objecten. De laag zal genegeerd worden in de (topo)logische validatie."
+                )
+            else:
+                datamodel.set_data(gdf, layer, index_col=None)
             syntax_result += [layer]
 
         # do logical validation: append result to layers_summary
         result_summary.status = "logical validation"
-        logger.info("start (topo)logical-validation")
+        logger.info("start (topo)logische validatie van object-lagen")
         layers_summary, result_summary = logical_validation.execute(
             datamodel,
             validation_rules_sets,
@@ -312,7 +325,7 @@ def _validator(
         )
 
         # finish validation and export results
-        logger.info("exporting results")
+        logger.info("exporteren resultaten")
         result_summary.status = "export results"
         result_layers = layers_summary.export(results_path, output_types)
         result_summary.result_layers = result_layers
@@ -328,21 +341,21 @@ def _validator(
         result_summary.success = True
         result_summary.status = "finished"
         result_summary.duration = timer.report()
-        logger.info(f"finished in {result_summary.duration:.2f} seconds")
+        logger.info(f"klaar in {result_summary.duration:.2f} seconden")
 
         _log_to_results(log_file, result_summary)
         result_summary.to_json(results_path)
 
+        _close_log_file(logger)
+
         return datamodel, layers_summary, result_summary
 
     except Exception as e:
-        e_str = str(e).replace("\n", " ")
-        e_str = " ".join(e_str.split())
-        stacktrace = rf"{traceback.format_exc(limit=2)}".split("\n")
+        stacktrace = rf"\n{traceback.format_exc(limit=0, chain=False)}".split("\n")
         if result_summary.error is not None:
-            result_summary.error += [stacktrace]
+            result_summary.error += stacktrace
         else:
-            result_summary.error = [stacktrace]
+            result_summary.error = stacktrace
         if results_path is not None:
             result_layers = layers_summary.export(results_path, output_types)
             _log_to_results(log_file, result_summary)
@@ -352,4 +365,6 @@ def _validator(
         else:
             result_summary.to_dict()
 
+        _close_log_file(logger)
+
         return None

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b5}/hydamo_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydamo_validation
-Version: 1.3.0b3
+Version: 1.3.0b5
 Summary: Validation module for HyDAMO data
 Author-email: Daniel Tollenaar <daniel@d2hydro.nl>
 License: MIT

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b5}/pyproject.toml

@@ -1,46 +1,46 @@
-[build-system]
-requires = ["setuptools>=61"]
-build-backend = "setuptools.build_meta"
-
-[project]
-name = "hydamo_validation"
-description = "Validation module for HyDAMO data"
-readme = "README.md"
-authors = [
-    { name = "Daniel Tollenaar", email = "daniel@d2hydro.nl" },
-]
-license = { text = "MIT" }
-
-requires-python = ">=3.12"
-dependencies = [
-    "geopandas",
-    "pandas>=2",
-    "pyogrio",
-    "rasterio",
-    "shapely>=2",
-    "rasterstats",
-]
-
-dynamic = ["version"]
-
-[project.optional-dependencies]
-tests = ["pytest"]
-
-[tool.flake8]
-max-line-length = 120
-max-complexity = 10
-
-[tool.setuptools]
-zip-safe = true
-
-[tool.setuptools.dynamic]
-version = { attr = "hydamo_validation.__version__" }
-
-[tool.setuptools.packages.find]
-include = ["hydamo_validation", "hydamo_validation.*"]
-
-[tool.setuptools.package-data]
-hydamo_validation = ["schemas/hydamo/*.json","schemas/rules/*.json", "styles/*.*"]
-
-[project.urls]
-Source = "https://github.com/HetWaterschapshuis/HyDAMOValidatieModule"
+[build-system]
+requires = ["setuptools>=61"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hydamo_validation"
+description = "Validation module for HyDAMO data"
+readme = "README.md"
+authors = [
+    { name = "Daniel Tollenaar", email = "daniel@d2hydro.nl" },
+]
+license = { text = "MIT" }
+
+requires-python = ">=3.12"
+dependencies = [
+    "geopandas",
+    "pandas>=2",
+    "pyogrio",
+    "rasterio",
+    "shapely>=2",
+    "rasterstats",
+]
+
+dynamic = ["version"]
+
+[project.optional-dependencies]
+tests = ["pytest"]
+
+[tool.flake8]
+max-line-length = 120
+max-complexity = 10
+
+[tool.setuptools]
+zip-safe = true
+
+[tool.setuptools.dynamic]
+version = { attr = "hydamo_validation.__version__" }
+
+[tool.setuptools.packages.find]
+include = ["hydamo_validation", "hydamo_validation.*"]
+
+[tool.setuptools.package-data]
+hydamo_validation = ["schemas/hydamo/*.json","schemas/rules/*.json", "styles/*.*"]
+
+[project.urls]
+Source = "https://github.com/HetWaterschapshuis/HyDAMOValidatieModule"

hydamo_validation-1.3.0b5/tests/test_dommelerwaard.py

@@ -0,0 +1,87 @@
+# %%
+from hydamo_validation import validator
+from pathlib import Path
+import pandas as pd
+
+DATA_DIR = Path(r"d:\repositories\HyDAMOValidatieModule\tests\data")
+coverage = {"AHN": DATA_DIR.joinpath(r"dtm")}
+directory = DATA_DIR.joinpath(r"tasks/test_dommelerwaard")
+exports_dir = Path(__file__).parent / "exports"
+exports_dir.mkdir(exist_ok=True)
+
+hydamo_validator = validator(
+    output_types=["geopackage", "csv", "geojson"], coverages=coverage, log_level="INFO"
+)
+
+
+datamodel, layer_summary, result_summary = hydamo_validator(
+    directory=directory, raise_error=False
+)
+
+
+def test_hydroobjects():
+    EXPECTED_INDEX = pd.Index(
+        [
+            "nen3610id",
+            "code",
+            "geometry",
+            "syntax_breedte",
+            "syntax_categorieoppwaterlichaam",
+            "syntax_code",
+            "syntax_created_date",
+            "syntax_detailniveaugeometrie",
+            "syntax_draineert",
+            "syntax_featuretype",
+            "syntax_getijdebeinvloed",
+            "syntax_globalid",
+            "syntax_hydroobjectgroeperingid",
+            "syntax_hyperlink",
+            "syntax_krwstromendid",
+            "syntax_last_edited_date",
+            "syntax_lengte",
+            "syntax_lvpublicatiedatum",
+            "syntax_metadataid",
+            "syntax_naam",
+            "syntax_namespace",
+            "syntax_nen3610id",
+            "syntax_niveau",
+            "syntax_objectid",
+            "syntax_ontstaanswijze",
+            "syntax_opmerking",
+            "syntax_oppervlaktewaterlichaamid",
+            "syntax_oppwatervolgnummer",
+            "syntax_persistentie",
+            "syntax_ruimtelijkeafbakeningbekend",
+            "syntax_ruwheidhoog",
+            "syntax_ruwheidlaag",
+            "syntax_soortoppwaterkwaliteit",
+            "syntax_soortoppwaterkwantiteit",
+            "syntax_statusleggerwatersysteem",
+            "syntax_statusobject",
+            "syntax_typeruwheid",
+            "syntax_vaarwegid",
+            "syntax_geometry",
+            "syntax_oordeel",
+            "validate_000_lengte_gt_1m",
+            "validate_001_goed_gesplitst",
+            "validate_002_goede_kruising",
+            "validate_003_geen_eindpunt",
+            "validate_006_kunstwerk_op_peilscheiding",
+            "validate_005_kunstwerk_tussen_hobject",
+            "rating",
+            "valid",
+            "invalid",
+            "invalid_critical",
+            "invalid_non_critical",
+            "ignored",
+            "summary",
+            "tags_assigned",
+            "tags_invalid",
+            "date_check",
+        ]
+    )
+
+    assert layer_summary.hydroobject.columns.equals(EXPECTED_INDEX)
+
+
+# %%

hydamo_validation-1.3.0b3/tests/test_dommelerwaard.py

@@ -1,31 +0,0 @@
-# %%
-from hydamo_validation import validator
-from hydamo_validation import __version__
-from pathlib import Path
-from datetime import datetime
-
-# try:
-#     from .config import DATA_DIR
-# except ImportError:
-#     from config import DATA_DIR
-
-DATA_DIR = Path(r"d:\repositories\HyDAMOValidatieModule\tests\data")
-coverage = {"AHN": DATA_DIR.joinpath(r"dtm")}
-directory = DATA_DIR.joinpath(r"tasks/test_dommelerwaard")
-exports_dir = Path(__file__).parent / "exports"
-exports_dir.mkdir(exist_ok=True)
-
-# hydamo_validator = validator(output_types=["geopackage", "csv", "geojson"],
-#                              coverages=coverage,
-#                              log_level="INFO"
-#                              )
-
-
-# datamodel, layer_summary, result_summary = hydamo_validator(
-#     directory=directory,
-#     raise_error=True
-#     )
-
-# datamodel.to_geopackage(exports_dir / "datamodel.gpkg",
-#                         use_schema=False)
-# %%