hydamo-validation 1.3.0b3__tar.gz → 1.3.0b4__tar.gz

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydamo_validation
-Version: 1.3.0b3
+Version: 1.3.0b4
 Summary: Validation module for HyDAMO data
 Author-email: Daniel Tollenaar <daniel@d2hydro.nl>
 License: MIT

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b4}/hydamo_validation/__init__.py

@@ -1,13 +1,14 @@
 __author__ = ["Het Waterschapshuis", "D2HYDRO", "HKV", "HydroConsult"]
 __copyright__ = "Copyright 2021, HyDAMO ValidatieTool"
 __credits__ = ["D2HYDRO", "HKV", "HydroConsult"]
-__version__ = "1.3.0b3"
+__version__ = "1.3.0b4"
 
 __license__ = "MIT"
 __maintainer__ = "Daniel Tollenaar"
 __email__ = "daniel@d2hydro.nl"
 
-import fiona
+import fiona  # top-level import to avoid fiona import issue: https://github.com/conda-forge/fiona-feedstock/issues/213
+from pathlib import Path
 from hydamo_validation.functions import topologic as topologic_functions
 from hydamo_validation.functions import logic as logic_functions
 from hydamo_validation.functions import general as general_functions
@@ -19,4 +20,5 @@ __all__ = [
     "logic_functions",
     "general_functions",
     "validator",
+    "SCHEMAS_PATH",
 ]

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b4}/hydamo_validation/logical_validation.py

@@ -127,11 +127,14 @@ def execute(
         for i in validation_rules_sets["objects"]
         if i["object"] in datamodel.data_layers
     )
+    logger.info(
+        rf"lagen met valide objecten én regels: {[i["object"] for i in object_rules_sets]}"
+    )
     for object_rules in object_rules_sets:
         col_translation: dict = {}
 
         object_layer = object_rules["object"]
-        logger.info(f"start logical validations of layer: {object_layer}")
+        logger.info(f"{object_layer}: start")
         object_gdf = getattr(datamodel, object_layer).copy()
 
         # add summary columns
@@ -145,7 +148,7 @@ def execute(
             general_rules_sorted = sorted(general_rules, key=lambda k: k["id"])
             for rule in general_rules_sorted:
                 logger.info(
-                    f"{object_layer}: executing general-rule with id {rule['id']}"
+                    f"{object_layer}: uitvoeren general-rule met id {rule['id']}"
                 )
                 try:
                     result_variable = rule["result_variable"]
@@ -224,7 +227,7 @@ def execute(
             try:
                 rule_id = rule["id"]
                 logger.info(
-                    f"{object_layer}: executing validation-rule with id {rule_id} ({rule['name']})"
+                    f"{object_layer}: uitvoeren validatieregel met id {rule_id} ({rule['name']})"
                 )
                 result_variable = rule["result_variable"]
                 if "exceptions" in rule.keys():

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b4}/hydamo_validation/summaries.py

@@ -142,7 +142,7 @@ class LayersSummary:
                         gdf_out = gdf.copy()
                         if gdf_out.crs:
                             gdf_out.to_crs("epsg:4326", inplace=True)
-                        gdf_out.to_file(file_path, driver="GeoJSON")
+                        gdf_out.to_file(file_path, driver="GeoJSON", engine="pyogrio")
 
                     # drop geometry for writing to csv
                     elif output_type == "csv":
@@ -150,7 +150,7 @@ class LayersSummary:
                             output_type, f"{object_layer}.csv"
                         )
                         df = gdf.drop("geometry", axis=1)
-                        df.to_csv(file_path)
+                        df.to_csv(file_path, index=False)
 
                     # write to geopackage as is
                     elif output_type == "geopackage":
@@ -182,7 +182,7 @@ class ResultSummary:
         self.error_layers = []
         self.syntax_result = []
         self.validation_result = []
-        self.error = None
+        self.error = []
         self.errors = None
         self.warnings = None
 

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b4}/hydamo_validation/validator.py

@@ -23,7 +23,7 @@ import traceback
 
 OUTPUT_TYPES = ["geopackage", "geojson", "csv"]
 LOG_LEVELS = Literal["INFO", "DEBUG"]
-INCLUDE_COLUMNS = ["code"]
+INCLUDE_COLUMNS = ["nen3610id", "code"]
 SCHEMAS_PATH = Path(__file__).parent.joinpath(r"./schemas")
 HYDAMO_SCHEMAS_PATH = SCHEMAS_PATH.joinpath("hydamo")
 RULES_SCHEMAS_PATH = SCHEMAS_PATH.joinpath("rules")
@@ -54,6 +54,13 @@ def _add_log_file(logger, log_file):
     return logger
 
 
+def _close_log_file(logger):
+    """Remove log-file from existing logger"""
+    for h in logger.handlers:
+        h.close()
+        logger.removeHandler(h)
+
+
 def _log_to_results(log_file, result_summary):
     result_summary.log = log_file.read_text().split("\n")
 
@@ -180,28 +187,31 @@ def _validator(
             log_level=log_level,
         )
 
-        logger.info("validator start")
+        logger.info("init validatie")
         date_check = pd.Timestamp.now().isoformat()
         result_summary = ResultSummary(date_check=date_check)
         layers_summary = LayersSummary(date_check=date_check)
         # check if all files are present
         # create a results_path
+        permission_error = False
         if dir_path.exists():
             results_path = dir_path.joinpath("results")
             if results_path.exists():
                 try:
                     shutil.rmtree(results_path)
                 except PermissionError:
-                    logger.warning(
-                        f"Cannot remove {results_path}. This may lead to write-issues later!"
-                    )
-                    pass
+                    permission_error = True
             results_path.mkdir(parents=True, exist_ok=True)
         else:
             raise FileNotFoundError(f"{dir_path.absolute().resolve()} does not exist")
 
         log_file = results_path.joinpath("validator.log")
         logger = _add_log_file(logger, log_file=log_file)
+        logger.info("start validatie")
+        if permission_error:
+            logger.warning(
+                f"Kan pad {results_path} niet verwijderen. Dit kan later tot problemen leiden!"
+            )
         dataset_path = dir_path.joinpath("datasets")
         validation_rules_json = dir_path.joinpath("validationrules.json")
         missing_paths = []
@@ -209,11 +219,11 @@ def _validator(
             if not path.exists():
                 missing_paths += [str(path)]
         if missing_paths:
-            result_summary.error = [f'missing_paths: {",".join(missing_paths)}']
+            result_summary.error += [f'missing_paths: {",".join(missing_paths)}']
             raise FileNotFoundError(f'missing_paths: {",".join(missing_paths)}')
         else:
             validation_rules_sets = read_validation_rules(
-                validation_rules_json, SCHEMAS_PATH
+                validation_rules_json, result_summary
             )
 
         # check if output-files are supported
@@ -224,7 +234,7 @@ def _validator(
             error_message = (
                 r"unsupported output types: " f'{",".join(unsupported_output_types)}'
             )
-            result_summary.error = [error_message]
+            result_summary.error += [error_message]
             raise TypeError(error_message)
 
         # set coverages
@@ -249,7 +259,7 @@ def _validator(
         result_summary.dataset_layers = datasets.layers
 
         ## validate syntax of datasets on layers-level and append to result
-        logger.info("start syntax-validation of object-layers")
+        logger.info("start syntax-validatie van object-lagen")
         valid_layers = datamodel_layers(datamodel.layers, datasets.layers)
         result_summary.missing_layers = missing_layers(
             datamodel.layers, datasets.layers
@@ -265,7 +275,7 @@ def _validator(
             status_object = validation_rules_sets["status_object"]
 
         for layer in valid_layers:
-            logger.info(f"read layer {layer}")
+            logger.info(f"{layer}: inlezen")
 
             # read layer
             gdf, schema = datasets.read_layer(
@@ -275,7 +285,7 @@ def _validator(
                 gdf.empty
             ):  # pass if gdf is empty. Most likely due to mall-formed or ill-specifiec status_object
                 logger.warning(
-                    f"layer {layer} is empty. Be aware only values of {status_object} in field 'statusobject' are read!"
+                    f"{layer}: geen objecten ingelezen. Zorg dat alle waarden in de kolom 'status_object' voorkomen in {status_object}"
                 )
                 continue
 
@@ -285,23 +295,28 @@ def _validator(
                     gdf[col] = None
                     schema["properties"][col] = "str"
 
-            logger.info(f"syntax-validation of fields in layer {layer}")
+            logger.info(f"{layer}: syntax-validatie")
             gdf, result_gdf = fields_syntax(
                 gdf,
-                schema,
-                datamodel.validation_schemas[layer],
+                schema=schema,
+                validation_schema=datamodel.validation_schemas[layer],
                 keep_columns=INCLUDE_COLUMNS,
             )
 
             # Add the syntax-validation result to the results_summary
             layers_summary.set_data(result_gdf, layer, schema["geometry"])
             # Add the corrected datasets_layer data to the datamodel.
-            datamodel.set_data(gdf, layer, index_col=None)
+            if gdf.empty:
+                logger.warning(
+                    f"{layer}: geen valide objecten na syntax-validatie. Inspecteer 'syntax_oordeel' in de resultaten; deze is false voor alle objecten. De laag zal genegeerd worden in de (topo)logische validatie."
+                )
+            else:
+                datamodel.set_data(gdf, layer, index_col=None)
             syntax_result += [layer]
 
         # do logical validation: append result to layers_summary
         result_summary.status = "logical validation"
-        logger.info("start (topo)logical-validation")
+        logger.info("start (topo)logische validatie van object-lagen")
         layers_summary, result_summary = logical_validation.execute(
             datamodel,
             validation_rules_sets,
@@ -312,7 +327,7 @@ def _validator(
         )
 
         # finish validation and export results
-        logger.info("exporting results")
+        logger.info("exporteren resultaten")
         result_summary.status = "export results"
         result_layers = layers_summary.export(results_path, output_types)
         result_summary.result_layers = result_layers
@@ -328,21 +343,21 @@ def _validator(
         result_summary.success = True
         result_summary.status = "finished"
         result_summary.duration = timer.report()
-        logger.info(f"finished in {result_summary.duration:.2f} seconds")
+        logger.info(f"klaar in {result_summary.duration:.2f} seconden")
 
         _log_to_results(log_file, result_summary)
         result_summary.to_json(results_path)
 
+        _close_log_file(logger)
+
         return datamodel, layers_summary, result_summary
 
     except Exception as e:
-        e_str = str(e).replace("\n", " ")
-        e_str = " ".join(e_str.split())
-        stacktrace = rf"{traceback.format_exc(limit=2)}".split("\n")
+        stacktrace = rf"\n{traceback.format_exc(limit=0, chain=False)}".split("\n")
         if result_summary.error is not None:
-            result_summary.error += [stacktrace]
+            result_summary.error += stacktrace
         else:
-            result_summary.error = [stacktrace]
+            result_summary.error = stacktrace
         if results_path is not None:
             result_layers = layers_summary.export(results_path, output_types)
             _log_to_results(log_file, result_summary)
@@ -352,4 +367,6 @@ def _validator(
         else:
             result_summary.to_dict()
 
+        _close_log_file(logger)
+
         return None

{hydamo_validation-1.3.0b3 → hydamo_validation-1.3.0b4}/hydamo_validation.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: hydamo_validation
-Version: 1.3.0b3
+Version: 1.3.0b4
 Summary: Validation module for HyDAMO data
 Author-email: Daniel Tollenaar <daniel@d2hydro.nl>
 License: MIT