data-validation-engine 0.8.3__tar.gz → 0.8.5__tar.gz

{data_validation_engine-0.8.3 → data_validation_engine-0.8.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-validation-engine
-Version: 0.8.3
+Version: 0.8.5
 Summary: `nhs data validation engine` is a framework used to validate data
 License-Expression: MIT
 License-File: LICENSE

{data_validation_engine-0.8.3 → data_validation_engine-0.8.5}/pyproject.toml

@@ -25,7 +25,7 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
 Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
 
 [tool.poetry]
-version = "0.8.3"
+version = "0.8.5"
 packages = [
     { include = "dve", from = "src" },
 ]
@@ -69,7 +69,7 @@ optional = true
 behave = "1.3.3"
 coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.2.14"}
-cryptography = "48.0.1"  # dependency of `moto`
+cryptography = "50.0.0"  # dependency of `moto`
 requests = "2.33.0"  # dependency of `moto`
 Werkzeug = "3.1.6"
 pytest = "9.0.3"
@@ -106,7 +106,7 @@ click = "8.2.1"
 mkdocs = "1.6.1"
 mkdocstrings = { version = "1.0.3", extras = ["python"] }
 griffelib = "2.0.1"
-pymdown-extensions = "10.21.3"
+pymdown-extensions = "11.0.1"
 zensical = "0.0.46"
 
 [tool.ruff]

{data_validation_engine-0.8.3 → data_validation_engine-0.8.5}/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py

@@ -24,6 +24,7 @@ from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type
 from dve.core_engine.backends.utilities import DEFAULT_ISO_FORMATS, datetime_format_to_regex
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
 from dve.core_engine.type_hints import URI, EntityName
+from dve.metadata_parser.utilities import resilient_get
 from dve.parser.file_handling.service import LocalFilesystemImplementation, _get_implementation
 
 
@@ -451,23 +452,27 @@ def get_duckdb_cast_statement_from_annotation(
         raise ValueError(f"dict must be `typing.TypedDict` subclass, got {type_annotation!r}")
 
     for type_ in type_annotation.mro():
-        _date_format: str = getattr(  # type: ignore
-            type_, "DATE_FORMAT", DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(datetime))
-        )
-        dt_cast_statement = rf"CASE WHEN REGEXP_FULL_MATCH(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_STRPTIME(TRIM({quoted_name}), '{_date_format}') ELSE NULL END"  # pylint: disable=C0301
-
-        # datetime is subclass of date, so needs to be handled first
-        if issubclass(type_, datetime):
-            stmt = rf"TRY_CAST({dt_cast_statement} as TIMESTAMP)"
-            return stmt
-        if issubclass(type_, date):
-            stmt = rf"TRY_CAST({dt_cast_statement} as DATE)"
-            return stmt
-        if issubclass(type_, time):
-            stmt = rf"TRY_CAST({dt_cast_statement} as TIME)"
-            return stmt
-        duck_type = get_duckdb_type_from_annotation(type_)
-        if duck_type:
-            stmt = f"TRIM({quoted_name})"
-            return _cast_as_ddb_type(stmt, type_) if parent_element else stmt
+        if issubclass(type_, (date, time)):
+            _date_format: str = resilient_get(
+                type_, "DATE_FORMAT", "TIME_FORMAT"
+            ) or DEFAULT_ISO_FORMATS.get(
+                type_, DEFAULT_ISO_FORMATS.get(datetime)
+            )  # type: ignore
+            dt_cast_statement = rf"CASE WHEN REGEXP_FULL_MATCH(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_STRPTIME(TRIM({quoted_name}), '{_date_format}') ELSE NULL END"  # pylint: disable=C0301
+
+            # datetime is subclass of date, so needs to be handled first
+            if issubclass(type_, datetime):
+                stmt = rf"TRY_CAST({dt_cast_statement} as TIMESTAMP)"
+                return stmt
+            if issubclass(type_, date):
+                stmt = rf"TRY_CAST({dt_cast_statement} as DATE)"
+                return stmt
+            if issubclass(type_, time):
+                stmt = rf"TRY_CAST({dt_cast_statement} as TIME)"
+                return stmt
+        else:
+            duck_type = get_duckdb_type_from_annotation(type_)
+            if duck_type:
+                stmt = f"TRIM({quoted_name})"
+                return _cast_as_ddb_type(stmt, type_) if parent_element else stmt
     raise ValueError(f"No equivalent DuckDB type for {type_annotation!r}")

{data_validation_engine-0.8.3 → data_validation_engine-0.8.5}/src/dve/core_engine/backends/implementations/spark/spark_helpers.py

@@ -593,29 +593,31 @@ def get_spark_cast_statement_from_annotation(
         raise ValueError(f"dict must be `typing.TypedDict` subclass, got {type_annotation!r}")
 
     for type_ in type_annotation.mro():
-        _date_format: str = getattr(  # type: ignore
-            type_,
-            "DATE_FORMAT",
-            DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(dt.datetime)),
-        )
-
-        # pylint: disable=C0301
-        dt_cast_statement = f"CASE WHEN REGEXP(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_TO_TIMESTAMP(TRIM({quoted_name}), \"{python_to_java_datetime_format(_date_format)}\") ELSE NULL END"  # pylint: disable=C0301
-        # datetime is subclass of date, so needs to be handled first
-        if issubclass(type_, dt.datetime):
-            return (
-                _cast_as_spark_type(dt_cast_statement, type_)
-                if parent_element
-                else dt_cast_statement
-            )
         if issubclass(type_, dt.date):
-            return (
-                _cast_as_spark_type(dt_cast_statement, type_)
-                if parent_element
-                else dt_cast_statement
+            _date_format: str = getattr(  # type: ignore
+                type_,
+                "DATE_FORMAT",
+                DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(dt.datetime)),
             )
-        spark_type = get_type_from_annotation(type_)
-        if spark_type:
-            stmt = f"TRIM({quoted_name})"
-            return _cast_as_spark_type(stmt, type_) if parent_element else stmt
+
+            # pylint: disable=C0301
+            dt_cast_statement = f"CASE WHEN REGEXP(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_TO_TIMESTAMP(TRIM({quoted_name}), \"{python_to_java_datetime_format(_date_format)}\") ELSE NULL END"  # pylint: disable=C0301
+            # datetime is subclass of date, so needs to be handled first
+            if issubclass(type_, dt.datetime):
+                return (
+                    _cast_as_spark_type(dt_cast_statement, type_)
+                    if parent_element
+                    else dt_cast_statement
+                )
+            if issubclass(type_, dt.date):
+                return (
+                    _cast_as_spark_type(dt_cast_statement, type_)
+                    if parent_element
+                    else dt_cast_statement
+                )
+        else:
+            spark_type = get_type_from_annotation(type_)
+            if spark_type:
+                stmt = f"TRIM({quoted_name})"
+                return _cast_as_spark_type(stmt, type_) if parent_element else stmt
     raise ValueError(f"No equivalent Spark type for {type_annotation!r}")

{data_validation_engine-0.8.3 → data_validation_engine-0.8.5}/src/dve/metadata_parser/utilities.py

@@ -52,3 +52,23 @@ def chain_get(
         return result
 
     raise exc.TypeNotFoundError(f"Callable or type ({item!r}) not found")
+
+
+def resilient_get(item: object, *attribute_names: str) -> Any:
+    """Given a number of attribute names, try to get attribute value
+    sequentially. Returns the first value found, and if no attributes found
+    returns None.
+
+    Args:
+        item (object): The object to obtain attributes from (where possible)
+        attribute_names (str): The attribute names to search for
+
+    Returns:
+        Any: The first found attribute, otherwise None
+    """
+    for attr in attribute_names:
+        try:
+            return getattr(item, attr)
+        except AttributeError:
+            continue
+    return None

{data_validation_engine-0.8.3 → data_validation_engine-0.8.5}/src/dve/pipeline/pipeline.py

@@ -852,7 +852,11 @@ class BaseDVEPipeline:
         summary_items = er.SummaryItems(
             submission_status=submission_status,
             summary_dict=summary_dict,
-            row_headings=[e.reporting_name for e in ErrorReportCategories],
+            row_headings=[
+                e.reporting_name
+                for e in ErrorReportCategories
+                if e != ErrorReportCategories.RECORD_REJECTION
+            ],
         )
 
         workbook = er.ExcelFormat(

{data_validation_engine-0.8.3 → data_validation_engine-0.8.5}/src/dve/reporting/excel_report.py

@@ -153,17 +153,6 @@ class SummaryItems:
                 ),  # pylint: disable=C0301
             ]
         )
-        if status not in (
-            ErrorReportStatus.PROCESSING_FAILED,
-            ErrorReportStatus.FILE_REJECTION,
-        ):
-            summary.append(
-                [
-                    "",
-                    "Total Number of Records Rejected",
-                    self.submission_status.number_of_records_rejected,
-                ]
-            )
         summary.append(["", ""])