data-validation-engine 0.8.1__tar.gz → 0.8.3__tar.gz

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-validation-engine
-Version: 0.8.1
+Version: 0.8.3
 Summary: `nhs data validation engine` is a framework used to validate data
 License-Expression: MIT
 License-File: LICENSE
@@ -10,6 +10,7 @@ Requires-Python: >=3.10,<3.13
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Typing :: Typed
@@ -25,7 +26,7 @@ Requires-Dist: pandas (==2.3.3)
 Requires-Dist: polars (==0.20.31)
 Requires-Dist: pyarrow (==23.0.1)
 Requires-Dist: pydantic (==1.10.19)
-Requires-Dist: pyspark (>=3.0.0,<=3.5.2)
+Requires-Dist: pyspark (>=3.5.0,<=3.5.5)
 Requires-Dist: typing_extensions (==4.15.0)
 Requires-Dist: urllib3 (==2.7.0)
 Project-URL: Changelog, https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/pyproject.toml

@@ -10,6 +10,7 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
     "Operating System :: OS Independent",
     "Topic :: Software Development :: Libraries",
     "Typing :: Typed",
@@ -24,7 +25,7 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
 Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
 
 [tool.poetry]
-version = "0.8.1"
+version = "0.8.3"
 packages = [
     { include = "dve", from = "src" },
 ]
@@ -43,7 +44,7 @@ pandas = "2.3.3"
 polars = "0.20.31"
 pyarrow = "23.0.1"
 pydantic = "1.10.19"
-pyspark = ">=3.0.0,<=3.5.2"
+pyspark = ">=3.5.0,<=3.5.5"
 typing_extensions = "4.15.0"
 urllib3 = "2.7.0"  # dependency of boto3 & botocore
 
@@ -68,6 +69,7 @@ optional = true
 behave = "1.3.3"
 coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.2.14"}
+cryptography = "48.0.1"  # dependency of `moto`
 requests = "2.33.0"  # dependency of `moto`
 Werkzeug = "3.1.6"
 pytest = "9.0.3"
@@ -78,7 +80,7 @@ xlsx2csv = "0.8.4"  # polars requirement
 optional = true
 
 [tool.poetry.group.lint.dependencies]
-black = "24.3.0"
+black = "26.3.1"
 astroid = "3.3.9"
 isort = "5.13.2"
 pylint = "3.3.9"
@@ -105,7 +107,7 @@ mkdocs = "1.6.1"
 mkdocstrings = { version = "1.0.3", extras = ["python"] }
 griffelib = "2.0.1"
 pymdown-extensions = "10.21.3"
-zensical = "0.0.31"
+zensical = "0.0.46"
 
 [tool.ruff]
 line-length = 100

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/base/auditing.py

@@ -1,5 +1,6 @@
 # pylint: disable=C0209
 """Base auditing objects and managers for use of DVE services"""
+
 import multiprocessing
 import operator
 import threading

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/base/reader.py

@@ -119,10 +119,7 @@ class BaseFileReader(ABC):
         """
         if entity_name == Iterator[dict[str, Any]]:
             return self.read_to_py_iterator(
-                resource,
-                entity_name,
-                schema, # type: ignore
-                all_model_fields
+                resource, entity_name, schema, all_model_fields  # type: ignore
             )
 
         self.raise_if_not_sensible_file(resource, entity_name)
@@ -133,11 +130,7 @@ class BaseFileReader(ABC):
             raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
 
         return reader_func(
-            self,
-            resource,
-            entity_name,
-            schema,
-            all_model_fields=all_model_fields  # type: ignore
+            self, resource, entity_name, schema, all_model_fields=all_model_fields  # type: ignore
         )
 
     def add_record_index(self, entity: EntityType, **kwargs) -> EntityType:

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/base/rules.py

@@ -681,3 +681,13 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
     def write_parquet(self, entity: EntityType, target_location: URI, **kwargs) -> URI:
         """Method to write parquet files"""
         raise NotImplementedError()
+
+    def filter_data_contract_record_rejections(
+        self,
+        working_directory: URI,
+        entity: EntityType,
+        entity_name: EntityName,
+        **kwargs,
+    ):
+        """Method to filter out record rejection errors from the data contract for a given entity"""
+        raise NotImplementedError()

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/exceptions.py

@@ -37,10 +37,7 @@ class UnableToParseCSVError(MessageBearingError):
     """An error raised when unable to parse a CSV file"""
 
     def __init__(
-        self,
-        entity_name: str,
-        field_check_error_message: str,
-        field_check_error_code: str
+        self, entity_name: str, field_check_error_message: str, field_check_error_code: str
     ):
         super().__init__(
             messages=[

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/duckdb/auditing.py

@@ -119,10 +119,8 @@ class DDBAuditor(BaseAuditor[DuckDBPyRelation]):
             schema=self.polars_schema,
         )
 
-        self._connection.sql(
-            f"""INSERT INTO {self._name} ({', '.join(self.polars_schema)})
-                                 SELECT {', '.join(self.polars_schema)} from data_pl_df"""
-        )
+        self._connection.sql(f"""INSERT INTO {self._name} ({', '.join(self.polars_schema)})
+                                 SELECT {', '.join(self.polars_schema)} from data_pl_df""")
 
     def retrieve_records(
         self,

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py

@@ -2,6 +2,7 @@
 # ignore: type[attr-defined]
 
 """Helper objects for duckdb data contract implementation"""
+
 from collections.abc import Generator, Iterator
 from dataclasses import is_dataclass
 from datetime import date, datetime, time
@@ -18,9 +19,11 @@ from pandas import DataFrame
 from pydantic import BaseModel
 from typing_extensions import Annotated, get_args, get_origin, get_type_hints
 
+from dve.common.error_utils import get_feedback_errors_uri
 from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type
+from dve.core_engine.backends.utilities import DEFAULT_ISO_FORMATS, datetime_format_to_regex
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.core_engine.type_hints import URI
+from dve.core_engine.type_hints import URI, EntityName
 from dve.parser.file_handling.service import LocalFilesystemImplementation, _get_implementation
 
 
@@ -100,7 +103,7 @@ def table_exists(connection: DuckDBPyConnection, table_name: str) -> bool:
 
 def relation_is_empty(relation: DuckDBPyRelation) -> bool:
     """Check if a duckdb relation is empty"""
-    if relation.limit(1).count("*"):
+    if relation.limit(1).shape[0] > 0:
         return False
     return True
 
@@ -256,6 +259,48 @@ def duckdb_write_parquet(cls):
     return cls
 
 
+def _ddb_filter_contract_errors(
+    self,
+    working_directory: URI,
+    entity: DuckDBPyRelation,
+    entity_name: EntityName,
+) -> DuckDBPyRelation:
+    contract_error_location = get_feedback_errors_uri(working_directory, "data_contract")
+    if not Path(contract_error_location).exists():
+        return entity
+    relevant_record_rejection_codes_rel = (
+        self._connection.read_json(
+            contract_error_location,
+            columns={
+                "RecordIndex": "INTEGER",
+                "FailureType": "STRING",
+                "Status": "STRING",
+                "Entity": "STRING",
+            },
+        )
+        .filter(
+            f"FailureType == 'record' AND Status != 'informational' AND Entity = '{entity_name}'"
+        )  # pylint: disable=C0301
+        .select("RecordIndex")
+        .distinct()
+        .order("RecordIndex asc")
+    )
+
+    if relation_is_empty(relevant_record_rejection_codes_rel):
+        return entity
+
+    filtered_entity = entity.join(
+        relevant_record_rejection_codes_rel, condition="__record_index__ == RecordIndex", how="anti"
+    )
+    return filtered_entity
+
+
+def ddb_filter_contract_errors(cls):
+    """Class decorator to filter out records that failed casting and have record rejection scope"""
+    cls.filter_data_contract_record_rejections = _ddb_filter_contract_errors
+    return cls
+
+
 @staticmethod  # type: ignore
 def _duckdb_get_entity_count(entity: DuckDBPyRelation) -> int:
     """Method to obtain entity count from a persisted parquet entity"""
@@ -317,7 +362,7 @@ def duckdb_record_index(cls):
 
 def _cast_as_ddb_type(field_expr: str, type_annotation: Any) -> str:
     """Cast to Duck DB type"""
-    return f"""try_cast({field_expr} as {get_duckdb_type_from_annotation(type_annotation)})"""
+    return f"""TRY_CAST({field_expr} as {get_duckdb_type_from_annotation(type_annotation)})"""
 
 
 def _ddb_safely_quote_name(field_name: str) -> str:
@@ -334,9 +379,6 @@ def get_duckdb_cast_statement_from_annotation(
     element_name: str,
     type_annotation: Any,
     parent_element: bool = True,
-    date_regex: str = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
-    timestamp_regex: str = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}((\+|\-)[0-9]{2}:[0-9]{2})?$",  # pylint: disable=C0301
-    time_regex: str = r"^[0-9]{2}:[0-9]{2}:[0-9]{2}$",
 ) -> str:
     """Generate casting statements for duckdb relations from type annotations"""
     type_origin = get_origin(type_annotation)
@@ -347,19 +389,23 @@ def get_duckdb_cast_statement_from_annotation(
     if type_origin is Union:
         python_type = _get_non_heterogenous_type(get_args(type_annotation))
         return get_duckdb_cast_statement_from_annotation(
-            element_name, python_type, parent_element, date_regex, timestamp_regex
+            element_name,
+            python_type,
+            parent_element,
         )
 
     # Type hint is e.g. `List[str]`, check to ensure non-heterogenity.
     if type_origin is list or (isinstance(type_origin, type) and issubclass(type_origin, list)):
         element_type = _get_non_heterogenous_type(get_args(type_annotation))
-        stmt = f"list_transform({quoted_name}, x -> {get_duckdb_cast_statement_from_annotation('x',element_type, False, date_regex, timestamp_regex)})"  # pylint: disable=C0301
+        stmt = f"LIST_TRANSFORM({quoted_name}, x -> {get_duckdb_cast_statement_from_annotation('x',element_type, False,)})"  # pylint: disable=C0301
         return stmt if not parent_element else _cast_as_ddb_type(stmt, type_annotation)
 
     if type_origin is Annotated:
         python_type, *other_args = get_args(type_annotation)  # pylint: disable=unused-variable
         return get_duckdb_cast_statement_from_annotation(
-            element_name, python_type, parent_element, date_regex, timestamp_regex
+            element_name,
+            python_type,
+            parent_element,
         )  # add other expected params here
     # Ensure that we have a concrete type at this point.
     if not isinstance(type_annotation, type):
@@ -384,7 +430,9 @@ def get_duckdb_cast_statement_from_annotation(
                 continue
 
             fields[field_name] = get_duckdb_cast_statement_from_annotation(
-                f"{element_name}.{field_name}", field_annotation, False, date_regex, timestamp_regex
+                f"{element_name}.{field_name}",
+                field_annotation,
+                False,
             )
 
         if not fields:
@@ -392,7 +440,7 @@ def get_duckdb_cast_statement_from_annotation(
                 f"No type annotations in dict/dataclass type (got {type_annotation!r})"
             )
         cast_exprs = ",".join([f'"{nme}":= {stmt}' for nme, stmt in fields.items()])
-        stmt = f"struct_pack({cast_exprs})"
+        stmt = f"STRUCT_PACK({cast_exprs})"
         return stmt if not parent_element else _cast_as_ddb_type(stmt, type_annotation)
 
     if type_annotation is list:
@@ -403,18 +451,23 @@ def get_duckdb_cast_statement_from_annotation(
         raise ValueError(f"dict must be `typing.TypedDict` subclass, got {type_annotation!r}")
 
     for type_ in type_annotation.mro():
+        _date_format: str = getattr(  # type: ignore
+            type_, "DATE_FORMAT", DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(datetime))
+        )
+        dt_cast_statement = rf"CASE WHEN REGEXP_FULL_MATCH(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_STRPTIME(TRIM({quoted_name}), '{_date_format}') ELSE NULL END"  # pylint: disable=C0301
+
         # datetime is subclass of date, so needs to be handled first
         if issubclass(type_, datetime):
-            stmt = rf"CASE WHEN REGEXP_MATCHES(TRIM({quoted_name}), '{timestamp_regex}') THEN TRY_CAST(TRIM({quoted_name}) as TIMESTAMP) ELSE NULL END"  # pylint: disable=C0301
+            stmt = rf"TRY_CAST({dt_cast_statement} as TIMESTAMP)"
             return stmt
         if issubclass(type_, date):
-            stmt = rf"CASE WHEN REGEXP_MATCHES(TRIM({quoted_name}), '{date_regex}') THEN TRY_CAST(TRIM({quoted_name}) as DATE) ELSE NULL END"  # pylint: disable=C0301
+            stmt = rf"TRY_CAST({dt_cast_statement} as DATE)"
             return stmt
         if issubclass(type_, time):
-            stmt = rf"CASE WHEN REGEXP_MATCHES(TRIM({quoted_name}), '{time_regex}') THEN TRY_CAST(TRIM({quoted_name}) as TIME) ELSE NULL END"  # pylint: disable=C0301
+            stmt = rf"TRY_CAST({dt_cast_statement} as TIME)"
             return stmt
         duck_type = get_duckdb_type_from_annotation(type_)
         if duck_type:
-            stmt = f"trim({quoted_name})"
+            stmt = f"TRIM({quoted_name})"
             return _cast_as_ddb_type(stmt, type_) if parent_element else stmt
     raise ValueError(f"No equivalent DuckDB type for {type_annotation!r}")

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -71,7 +71,7 @@ class DuckDBCSVReader(CSVFileReader):
             quote_char=quotechar,
             field_check=field_check,
             field_check_error_code=field_check_error_code,
-            field_check_error_message=field_check_error_message
+            field_check_error_message=field_check_error_message,
         )
 
     def read_to_py_iterator(
@@ -254,7 +254,7 @@ class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader):
             resource=resource,
             entity_name=entity_name,
             schema=schema,
-            all_model_fields=all_model_fields
+            all_model_fields=all_model_fields,
         )
         entity = entity.select(StarExpression(exclude=[RECORD_INDEX_COLUMN_NAME])).distinct()
         no_records = entity.shape[0]

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/duckdb/reference_data.py

@@ -43,4 +43,4 @@ class DuckDBRefDataLoader(BaseRefDataLoader[DuckDBPyRelation]):
     @mark_refdata_file_extension("arrow")
     def load_arrow_file(self, uri: str) -> DuckDBPyRelation:
         """Load an arrow ipc file into a duckdb relation"""
-        return self.connection.from_arrow(ipc.open_stream(uri).read_all())  # type:ignore
+        return self.connection.from_arrow(ipc.open_stream(uri).read_all())  # type: ignore

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/duckdb/rules.py

@@ -22,6 +22,7 @@ from dve.core_engine.backends.base.rules import (
 from dve.core_engine.backends.exceptions import ConstraintError
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
     DDBStruct,
+    ddb_filter_contract_errors,
     duckdb_read_parquet,
     duckdb_record_index,
     duckdb_rel_to_dictionaries,
@@ -61,6 +62,7 @@ from dve.core_engine.type_hints import Messages
 @duckdb_record_index
 @duckdb_write_parquet
 @duckdb_read_parquet
+@ddb_filter_contract_errors
 class DuckDBStepImplementations(BaseStepImplementations[DuckDBPyRelation]):
     """An implementation of transformation steps in duckdb."""
 

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/spark/contract.py

@@ -7,6 +7,7 @@ from typing import Any, Optional
 from uuid import uuid4
 
 from pydantic import BaseModel
+from pydantic.fields import ModelField
 from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql import functions as sf
 from pyspark.sql.functions import col, lit
@@ -26,6 +27,7 @@ from dve.core_engine.backends.exceptions import (
 )
 from dve.core_engine.backends.implementations.spark.spark_helpers import (
     df_is_empty,
+    get_spark_cast_statement_from_annotation,
     get_type_from_annotation,
     spark_read_parquet,
     spark_record_index,
@@ -86,7 +88,7 @@ class SparkDataContract(BaseDataContract[DataFrame]):
             schema=get_type_from_annotation(schema),
         )
 
-    # pylint: disable=R0915
+    # pylint: disable=R0914,R0915
     def apply_data_contract(
         self,
         working_dir: URI,
@@ -102,6 +104,7 @@ class SparkDataContract(BaseDataContract[DataFrame]):
 
         successful = True
         for entity_name, record_df in entities.items():
+            entity_fields: dict[str, ModelField] = contract_metadata.schemas[entity_name].__fields__
             spark_schema = get_type_from_annotation(contract_metadata.schemas[entity_name])
             spark_schema.add(StructField(RECORD_INDEX_COLUMN_NAME, LongType()))
             if df_is_empty(record_df):
@@ -144,12 +147,21 @@ class SparkDataContract(BaseDataContract[DataFrame]):
                 self.logger.info(f"Data contract found {msg_count} issues in {entity_name}")
 
             try:
+                # TODO: will need to revisit in pydantic v2 bump as model field no longer available
                 record_df = record_df.select(
-                    [
-                        col(column.name).cast(column.dataType)
-                        for column in spark_schema
-                        if column.name in record_df.columns
-                    ]
+                    *[
+                        (
+                            get_spark_cast_statement_from_annotation(
+                                fld, mdl_field.annotation
+                            ).alias(fld)
+                            if fld in record_df.columns
+                            else lit(None).cast(
+                                get_type_from_annotation(mdl_field.annotation).alias(fld)
+                            )
+                        )
+                        for fld, mdl_field in entity_fields.items()
+                    ],
+                    col(RECORD_INDEX_COLUMN_NAME).cast(LongType()).alias(RECORD_INDEX_COLUMN_NAME),
                 )
             except Exception as err:  # pylint: disable=broad-except
                 successful = False

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/spark/rules.py

@@ -14,6 +14,7 @@ from dve.core_engine.backends.implementations.spark.spark_helpers import (
     create_udf,
     get_all_registered_udfs,
     object_to_spark_literal,
+    spark_filter_contract_errors,
     spark_read_parquet,
     spark_record_index,
     spark_write_parquet,
@@ -53,6 +54,7 @@ from dve.core_engine.type_hints import Messages
 @spark_record_index
 @spark_write_parquet
 @spark_read_parquet
+@spark_filter_contract_errors
 class SparkStepImplementations(BaseStepImplementations[DataFrame]):
     """An implementation of transformation steps in Apache Spark."""
 

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/implementations/spark/spark_helpers.py

@@ -12,6 +12,7 @@ from collections.abc import Callable, Generator, Iterator
 from dataclasses import dataclass, is_dataclass
 from decimal import Decimal
 from functools import wraps
+from pathlib import Path
 from typing import Any, ClassVar, Optional, TypeVar, Union, overload
 
 from delta.exceptions import ConcurrentAppendException, DeltaConcurrentModificationException
@@ -25,9 +26,11 @@ from pyspark.sql.functions import lit, udf
 from pyspark.sql.types import LongType, StructField, StructType
 from typing_extensions import Annotated, Protocol, TypedDict, get_args, get_origin, get_type_hints
 
+from dve.common.error_utils import get_feedback_errors_uri
 from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type
+from dve.core_engine.backends.utilities import DEFAULT_ISO_FORMATS, datetime_format_to_regex
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.core_engine.type_hints import URI
+from dve.core_engine.type_hints import URI, EntityName
 
 # It would be really nice if there was a more parameterisable
 # way of doing this.
@@ -97,6 +100,21 @@ PYTHON_TYPE_TO_SPARK_TYPE: dict[type, st.DataType] = {
 }
 """A mapping of Python types to the equivalent Spark types."""
 
+PYTHON_TO_JAVA_DT_FORMAT_MAP: dict[str, str] = {
+    "%Y": "yyyy",
+    "%y": "yy",
+    "%m": "MM",
+    "%d": "dd",
+    "%H": "HH",
+    "%M": "mm",
+    "%S": "ss",
+    "%z": "XX",
+    "%Z": "z",
+}
+"""A mapping of python to java datetime component formats. Not exhaustive but will hopefully support
+   all requirements.
+"""
+
 
 PydanticModel = TypeVar("PydanticModel", bound=BaseModel)
 """An Pydantic model."""
@@ -270,6 +288,13 @@ def create_udf(function: Callable) -> Callable:
     return udf(function, returnType=return_type)
 
 
+def python_to_java_datetime_format(datetime_fmt: str) -> str:
+    "Helper to convert python datetime formats to Java datetime formats"
+    for pt, jt in PYTHON_TO_JAVA_DT_FORMAT_MAP.items():
+        datetime_fmt = datetime_fmt.replace(pt, jt)
+    return datetime_fmt.replace("T", "'T'")
+
+
 SupportedBaseType = Union[str, int, bool, float, Decimal, dt.date, dt.datetime]
 """Supported base types for Spark literals."""
 SparkLiteralType = Union[  # type: ignore
@@ -365,6 +390,53 @@ def spark_write_parquet(cls):
     return cls
 
 
+def _spark_filter_contract_errors(
+    self,
+    working_directory: URI,
+    entity: DataFrame,
+    entity_name: EntityName,
+) -> DataFrame:
+    contract_error_location = get_feedback_errors_uri(working_directory, "data_contract")
+    if not Path(contract_error_location).exists():
+        return entity
+
+    relevant_record_rejections_codes_df = (
+        self.spark_session.read.json(
+            path=contract_error_location,
+            schema=st.StructType(
+                [
+                    st.StructField("RecordIndex", st.IntegerType()),
+                    st.StructField("FailureType", st.StringType()),
+                    st.StructField("Status", st.StringType()),
+                    st.StructField("Entity", st.StringType()),
+                ]
+            ),
+        )
+        .filter(
+            (sf.col("FailureType") == sf.lit("record"))
+            & (sf.col("Status") != sf.lit("informational"))
+            & (sf.col("Entity") == sf.lit(entity_name))
+        )
+        .distinct()
+        .orderBy(sf.asc(sf.col("RecordIndex")))
+        # todo - ^^ possibly relook at join strat. Does this help? Over prescriptive?
+    )
+    if df_is_empty(relevant_record_rejections_codes_df):
+        return entity
+    filtered_entity = entity.join(
+        relevant_record_rejections_codes_df,
+        on=entity.__record_index__ == relevant_record_rejections_codes_df.RecordIndex,
+        how="anti",
+    )
+    return filtered_entity
+
+
+def spark_filter_contract_errors(cls):
+    """Class decorator to filter out records that failed casting and have record rejection scope"""
+    cls.filter_data_contract_record_rejections = _spark_filter_contract_errors
+    return cls
+
+
 @staticmethod  # type: ignore
 def _spark_get_entity_count(entity: DataFrame) -> int:
     """Method to obtain entity count from a persisted parquet entity"""
@@ -457,11 +529,7 @@ def _spark_safely_quote_name(field_name: str) -> str:
 
 # pylint: disable=R0801
 def get_spark_cast_statement_from_annotation(
-    element_name: str,
-    type_annotation: Any,
-    parent_element: bool = True,
-    date_regex: str = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}$",
-    timestamp_regex: str = r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}((\\+|\\-)[0-9]{2}:[0-9]{2})?$",  # pylint: disable=C0301
+    element_name: str, type_annotation: Any, parent_element: bool = True
 ):
     """Generate casting statements for spark dataframes based on type annotations"""
     type_origin = get_origin(type_annotation)
@@ -471,20 +539,18 @@ def get_spark_cast_statement_from_annotation(
     # An `Optional` or `Union` type, check to ensure non-heterogenity.
     if type_origin is Union:
         python_type = _get_non_heterogenous_type(get_args(type_annotation))
-        return get_spark_cast_statement_from_annotation(
-            element_name, python_type, parent_element, date_regex, timestamp_regex
-        )
+        return get_spark_cast_statement_from_annotation(element_name, python_type, parent_element)
 
     # Type hint is e.g. `List[str]`, check to ensure non-heterogenity.
     if type_origin is list or (isinstance(type_origin, type) and issubclass(type_origin, list)):
         element_type = _get_non_heterogenous_type(get_args(type_annotation))
-        stmt = f"transform({quoted_name}, x -> {get_spark_cast_statement_from_annotation('x',element_type, False, date_regex, timestamp_regex)})"  # pylint: disable=C0301
+        stmt = f"TRANSFORM({quoted_name}, x -> {get_spark_cast_statement_from_annotation('x',element_type, False)})"  # pylint: disable=C0301
         return stmt if not parent_element else _cast_as_spark_type(stmt, type_annotation)
 
     if type_origin is Annotated:
         python_type, *_ = get_args(type_annotation)  # pylint: disable=unused-variable
         return get_spark_cast_statement_from_annotation(
-            element_name, python_type, parent_element, date_regex, timestamp_regex
+            element_name, python_type, parent_element
         )  # add other expected params here
     # Ensure that we have a concrete type at this point.
     if not isinstance(type_annotation, type):
@@ -509,7 +575,7 @@ def get_spark_cast_statement_from_annotation(
                 continue
 
             fields[field_name] = get_spark_cast_statement_from_annotation(
-                f"{element_name}.{field_name}", field_annotation, False, date_regex, timestamp_regex
+                f"{element_name}.{field_name}", field_annotation, False
             )
 
         if not fields:
@@ -517,7 +583,7 @@ def get_spark_cast_statement_from_annotation(
                 f"No type annotations in dict/dataclass type (got {type_annotation!r})"
             )
         cast_exprs = ",".join([f"{stmt} AS `{nme}`" for nme, stmt in fields.items()])
-        stmt = f"struct({cast_exprs})"
+        stmt = f"STRUCT({cast_exprs})"
         return stmt if not parent_element else _cast_as_spark_type(stmt, type_annotation)
     if type_annotation is list:
         raise ValueError(
@@ -527,15 +593,29 @@ def get_spark_cast_statement_from_annotation(
         raise ValueError(f"dict must be `typing.TypedDict` subclass, got {type_annotation!r}")
 
     for type_ in type_annotation.mro():
+        _date_format: str = getattr(  # type: ignore
+            type_,
+            "DATE_FORMAT",
+            DEFAULT_ISO_FORMATS.get(type_, DEFAULT_ISO_FORMATS.get(dt.datetime)),
+        )
+
+        # pylint: disable=C0301
+        dt_cast_statement = f"CASE WHEN REGEXP(TRIM({quoted_name}), '{datetime_format_to_regex(_date_format)}') THEN TRY_TO_TIMESTAMP(TRIM({quoted_name}), \"{python_to_java_datetime_format(_date_format)}\") ELSE NULL END"  # pylint: disable=C0301
         # datetime is subclass of date, so needs to be handled first
         if issubclass(type_, dt.datetime):
-            stmt = rf"CASE WHEN REGEXP(TRIM({quoted_name}), '{timestamp_regex}') THEN TRIM({quoted_name}) ELSE NULL END"  # pylint: disable=C0301
-            return _cast_as_spark_type(stmt, type_) if parent_element else stmt
+            return (
+                _cast_as_spark_type(dt_cast_statement, type_)
+                if parent_element
+                else dt_cast_statement
+            )
         if issubclass(type_, dt.date):
-            stmt = rf"CASE WHEN REGEXP(TRIM({quoted_name}), '{date_regex}') THEN TRIM({quoted_name}) ELSE NULL END"  # pylint: disable=C0301
-            return _cast_as_spark_type(stmt, type_) if parent_element else stmt
+            return (
+                _cast_as_spark_type(dt_cast_statement, type_)
+                if parent_element
+                else dt_cast_statement
+            )
         spark_type = get_type_from_annotation(type_)
         if spark_type:
-            stmt = f"trim({quoted_name})"
+            stmt = f"TRIM({quoted_name})"
             return _cast_as_spark_type(stmt, type_) if parent_element else stmt
     raise ValueError(f"No equivalent Spark type for {type_annotation!r}")

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/metadata/contract.py

@@ -44,7 +44,7 @@ class DataContractMetadata(BaseModel, frozen=True, arbitrary_types_allowed=True)
         """The per-entity schemas, as pydantic models."""
         if not self._schemas:
             for entity_name, validator in self.validators.items():
-                self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137
+                self._schemas[entity_name] = validator.model  # type: ignore # pylint: disable=E1137
         return self._schemas.copy()  # pylint: disable=E1101
 
     @root_validator(allow_reuse=True)

{data_validation_engine-0.8.1 → data_validation_engine-0.8.3}/src/dve/core_engine/backends/readers/csv.py

@@ -16,11 +16,11 @@ from dve.core_engine.backends.exceptions import (
     MissingHeaderError,
 )
 from dve.core_engine.backends.readers.utilities import (
-    raise_message_bearing_error_on_header_differences
+    get_all_model_fields,
+    raise_message_bearing_error_on_header_differences,
 )
 from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.core_engine.backends.readers.utilities import get_all_model_fields
 from dve.core_engine.type_hints import EntityName
 from dve.parser.file_handling import get_content_length, open_stream
 from dve.parser.file_handling.implementations.file import file_uri_to_local_path