data-validation-engine 0.8.1__tar.gz → 0.8.2__tar.gz

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: data-validation-engine
-Version: 0.8.1
+Version: 0.8.2
 Summary: `nhs data validation engine` is a framework used to validate data
 License-Expression: MIT
 License-File: LICENSE

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/pyproject.toml

@@ -24,7 +24,7 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
 Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
 
 [tool.poetry]
-version = "0.8.1"
+version = "0.8.2"
 packages = [
     { include = "dve", from = "src" },
 ]
@@ -68,6 +68,7 @@ optional = true
 behave = "1.3.3"
 coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.2.14"}
+cryptography = "48.0.1"  # dependency of `moto`
 requests = "2.33.0"  # dependency of `moto`
 Werkzeug = "3.1.6"
 pytest = "9.0.3"
@@ -78,7 +79,7 @@ xlsx2csv = "0.8.4"  # polars requirement
 optional = true
 
 [tool.poetry.group.lint.dependencies]
-black = "24.3.0"
+black = "26.3.1"
 astroid = "3.3.9"
 isort = "5.13.2"
 pylint = "3.3.9"

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/base/auditing.py

@@ -1,5 +1,6 @@
 # pylint: disable=C0209
 """Base auditing objects and managers for use of DVE services"""
+
 import multiprocessing
 import operator
 import threading

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/base/reader.py

@@ -119,10 +119,7 @@ class BaseFileReader(ABC):
         """
         if entity_name == Iterator[dict[str, Any]]:
             return self.read_to_py_iterator(
-                resource,
-                entity_name,
-                schema, # type: ignore
-                all_model_fields
+                resource, entity_name, schema, all_model_fields  # type: ignore
             )
 
         self.raise_if_not_sensible_file(resource, entity_name)
@@ -133,11 +130,7 @@ class BaseFileReader(ABC):
             raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
 
         return reader_func(
-            self,
-            resource,
-            entity_name,
-            schema,
-            all_model_fields=all_model_fields  # type: ignore
+            self, resource, entity_name, schema, all_model_fields=all_model_fields  # type: ignore
         )
 
     def add_record_index(self, entity: EntityType, **kwargs) -> EntityType:

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/base/rules.py

@@ -681,3 +681,13 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
     def write_parquet(self, entity: EntityType, target_location: URI, **kwargs) -> URI:
         """Method to write parquet files"""
         raise NotImplementedError()
+
+    def filter_data_contract_record_rejections(
+        self,
+        working_directory: URI,
+        entity: EntityType,
+        entity_name: EntityName,
+        **kwargs,
+    ):
+        """Method to filter out record rejection errors from the data contract for a given entity"""
+        raise NotImplementedError()

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/exceptions.py

@@ -37,10 +37,7 @@ class UnableToParseCSVError(MessageBearingError):
     """An error raised when unable to parse a CSV file"""
 
     def __init__(
-        self,
-        entity_name: str,
-        field_check_error_message: str,
-        field_check_error_code: str
+        self, entity_name: str, field_check_error_message: str, field_check_error_code: str
     ):
         super().__init__(
             messages=[

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/duckdb/auditing.py

@@ -119,10 +119,8 @@ class DDBAuditor(BaseAuditor[DuckDBPyRelation]):
             schema=self.polars_schema,
         )
 
-        self._connection.sql(
-            f"""INSERT INTO {self._name} ({', '.join(self.polars_schema)})
-                                 SELECT {', '.join(self.polars_schema)} from data_pl_df"""
-        )
+        self._connection.sql(f"""INSERT INTO {self._name} ({', '.join(self.polars_schema)})
+                                 SELECT {', '.join(self.polars_schema)} from data_pl_df""")
 
     def retrieve_records(
         self,

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/duckdb/duckdb_helpers.py

@@ -2,6 +2,7 @@
 # ignore: type[attr-defined]
 
 """Helper objects for duckdb data contract implementation"""
+
 from collections.abc import Generator, Iterator
 from dataclasses import is_dataclass
 from datetime import date, datetime, time
@@ -18,9 +19,10 @@ from pandas import DataFrame
 from pydantic import BaseModel
 from typing_extensions import Annotated, get_args, get_origin, get_type_hints
 
+from dve.common.error_utils import get_feedback_errors_uri
 from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.core_engine.type_hints import URI
+from dve.core_engine.type_hints import URI, EntityName
 from dve.parser.file_handling.service import LocalFilesystemImplementation, _get_implementation
 
 
@@ -100,7 +102,7 @@ def table_exists(connection: DuckDBPyConnection, table_name: str) -> bool:
 
 def relation_is_empty(relation: DuckDBPyRelation) -> bool:
     """Check if a duckdb relation is empty"""
-    if relation.limit(1).count("*"):
+    if relation.limit(1).shape[0] > 0:
         return False
     return True
 
@@ -256,6 +258,48 @@ def duckdb_write_parquet(cls):
     return cls
 
 
+def _ddb_filter_contract_errors(
+    self,
+    working_directory: URI,
+    entity: DuckDBPyRelation,
+    entity_name: EntityName,
+) -> DuckDBPyRelation:
+    contract_error_location = get_feedback_errors_uri(working_directory, "data_contract")
+    if not Path(contract_error_location).exists():
+        return entity
+    relevant_record_rejection_codes_rel = (
+        self._connection.read_json(
+            contract_error_location,
+            columns={
+                "RecordIndex": "INTEGER",
+                "FailureType": "STRING",
+                "Status": "STRING",
+                "Entity": "STRING",
+            },
+        )
+        .filter(
+            f"FailureType == 'record' AND Status != 'informational' AND Entity = '{entity_name}'"
+        )  # pylint: disable=C0301
+        .select("RecordIndex")
+        .distinct()
+        .order("RecordIndex asc")
+    )
+
+    if relation_is_empty(relevant_record_rejection_codes_rel):
+        return entity
+
+    filtered_entity = entity.join(
+        relevant_record_rejection_codes_rel, condition="__record_index__ == RecordIndex", how="anti"
+    )
+    return filtered_entity
+
+
+def ddb_filter_contract_errors(cls):
+    """Class decorator to filter out records that failed casting and have record rejection scope"""
+    cls.filter_data_contract_record_rejections = _ddb_filter_contract_errors
+    return cls
+
+
 @staticmethod  # type: ignore
 def _duckdb_get_entity_count(entity: DuckDBPyRelation) -> int:
     """Method to obtain entity count from a persisted parquet entity"""

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -71,7 +71,7 @@ class DuckDBCSVReader(CSVFileReader):
             quote_char=quotechar,
             field_check=field_check,
             field_check_error_code=field_check_error_code,
-            field_check_error_message=field_check_error_message
+            field_check_error_message=field_check_error_message,
         )
 
     def read_to_py_iterator(
@@ -254,7 +254,7 @@ class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader):
             resource=resource,
             entity_name=entity_name,
             schema=schema,
-            all_model_fields=all_model_fields
+            all_model_fields=all_model_fields,
         )
         entity = entity.select(StarExpression(exclude=[RECORD_INDEX_COLUMN_NAME])).distinct()
         no_records = entity.shape[0]

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/duckdb/reference_data.py

@@ -43,4 +43,4 @@ class DuckDBRefDataLoader(BaseRefDataLoader[DuckDBPyRelation]):
     @mark_refdata_file_extension("arrow")
     def load_arrow_file(self, uri: str) -> DuckDBPyRelation:
         """Load an arrow ipc file into a duckdb relation"""
-        return self.connection.from_arrow(ipc.open_stream(uri).read_all())  # type:ignore
+        return self.connection.from_arrow(ipc.open_stream(uri).read_all())  # type: ignore

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/duckdb/rules.py

@@ -22,6 +22,7 @@ from dve.core_engine.backends.base.rules import (
 from dve.core_engine.backends.exceptions import ConstraintError
 from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
     DDBStruct,
+    ddb_filter_contract_errors,
     duckdb_read_parquet,
     duckdb_record_index,
     duckdb_rel_to_dictionaries,
@@ -61,6 +62,7 @@ from dve.core_engine.type_hints import Messages
 @duckdb_record_index
 @duckdb_write_parquet
 @duckdb_read_parquet
+@ddb_filter_contract_errors
 class DuckDBStepImplementations(BaseStepImplementations[DuckDBPyRelation]):
     """An implementation of transformation steps in duckdb."""
 

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/spark/rules.py

@@ -14,6 +14,7 @@ from dve.core_engine.backends.implementations.spark.spark_helpers import (
     create_udf,
     get_all_registered_udfs,
     object_to_spark_literal,
+    spark_filter_contract_errors,
     spark_read_parquet,
     spark_record_index,
     spark_write_parquet,
@@ -53,6 +54,7 @@ from dve.core_engine.type_hints import Messages
 @spark_record_index
 @spark_write_parquet
 @spark_read_parquet
+@spark_filter_contract_errors
 class SparkStepImplementations(BaseStepImplementations[DataFrame]):
     """An implementation of transformation steps in Apache Spark."""
 

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/spark/spark_helpers.py

@@ -12,6 +12,7 @@ from collections.abc import Callable, Generator, Iterator
 from dataclasses import dataclass, is_dataclass
 from decimal import Decimal
 from functools import wraps
+from pathlib import Path
 from typing import Any, ClassVar, Optional, TypeVar, Union, overload
 
 from delta.exceptions import ConcurrentAppendException, DeltaConcurrentModificationException
@@ -25,9 +26,10 @@ from pyspark.sql.functions import lit, udf
 from pyspark.sql.types import LongType, StructField, StructType
 from typing_extensions import Annotated, Protocol, TypedDict, get_args, get_origin, get_type_hints
 
+from dve.common.error_utils import get_feedback_errors_uri
 from dve.core_engine.backends.base.utilities import _get_non_heterogenous_type
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.core_engine.type_hints import URI
+from dve.core_engine.type_hints import URI, EntityName
 
 # It would be really nice if there was a more parameterisable
 # way of doing this.
@@ -365,6 +367,53 @@ def spark_write_parquet(cls):
     return cls
 
 
+def _spark_filter_contract_errors(
+    self,
+    working_directory: URI,
+    entity: DataFrame,
+    entity_name: EntityName,
+) -> DataFrame:
+    contract_error_location = get_feedback_errors_uri(working_directory, "data_contract")
+    if not Path(contract_error_location).exists():
+        return entity
+
+    relevant_record_rejections_codes_df = (
+        self.spark_session.read.json(
+            path=contract_error_location,
+            schema=st.StructType(
+                [
+                    st.StructField("RecordIndex", st.IntegerType()),
+                    st.StructField("FailureType", st.StringType()),
+                    st.StructField("Status", st.StringType()),
+                    st.StructField("Entity", st.StringType()),
+                ]
+            ),
+        )
+        .filter(
+            (sf.col("FailureType") == sf.lit("record"))
+            & (sf.col("Status") != sf.lit("informational"))
+            & (sf.col("Entity") == sf.lit(entity_name))
+        )
+        .distinct()
+        .orderBy(sf.asc(sf.col("RecordIndex")))
+        # todo - ^^ possibly relook at join strat. Does this help? Over prescriptive?
+    )
+    if df_is_empty(relevant_record_rejections_codes_df):
+        return entity
+    filtered_entity = entity.join(
+        relevant_record_rejections_codes_df,
+        on=entity.__record_index__ == relevant_record_rejections_codes_df.RecordIndex,
+        how="anti",
+    )
+    return filtered_entity
+
+
+def spark_filter_contract_errors(cls):
+    """Class decorator to filter out records that failed casting and have record rejection scope"""
+    cls.filter_data_contract_record_rejections = _spark_filter_contract_errors
+    return cls
+
+
 @staticmethod  # type: ignore
 def _spark_get_entity_count(entity: DataFrame) -> int:
     """Method to obtain entity count from a persisted parquet entity"""

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/metadata/contract.py

@@ -44,7 +44,7 @@ class DataContractMetadata(BaseModel, frozen=True, arbitrary_types_allowed=True)
         """The per-entity schemas, as pydantic models."""
         if not self._schemas:
             for entity_name, validator in self.validators.items():
-                self._schemas[entity_name] = validator.model # type: ignore # pylint: disable=E1137
+                self._schemas[entity_name] = validator.model  # type: ignore # pylint: disable=E1137
         return self._schemas.copy()  # pylint: disable=E1101
 
     @root_validator(allow_reuse=True)

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/readers/csv.py

@@ -16,11 +16,11 @@ from dve.core_engine.backends.exceptions import (
     MissingHeaderError,
 )
 from dve.core_engine.backends.readers.utilities import (
-    raise_message_bearing_error_on_header_differences
+    get_all_model_fields,
+    raise_message_bearing_error_on_header_differences,
 )
 from dve.core_engine.backends.utilities import get_polars_type_from_annotation, stringify_model
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.core_engine.backends.readers.utilities import get_all_model_fields
 from dve.core_engine.type_hints import EntityName
 from dve.parser.file_handling import get_content_length, open_stream
 from dve.parser.file_handling.implementations.file import file_uri_to_local_path

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/readers/utilities.py

@@ -44,16 +44,16 @@ def raise_message_bearing_error_on_header_differences(
     header or vice versa.
     """
     missing, additional = check_csv_header_expected(
-        resource,
-        expected_schema,
-        all_model_fields,
-        delimiter,
-        quote_char
+        resource, expected_schema, all_model_fields, delimiter, quote_char
     )
 
     if missing or additional:
-        record_details_missing = f"missing fields: {', '.join(sorted(missing))};" if missing else ""  # pylint: disable=C0301
-        record_details_additional = f"additional fields: {', '.join(sorted(additional))};" if additional else ""  # pylint: disable=C0301
+        record_details_missing = (
+            f"missing fields: {', '.join(sorted(missing))};" if missing else ""
+        )  # pylint: disable=C0301
+        record_details_additional = (
+            f"additional fields: {', '.join(sorted(additional))};" if additional else ""
+        )  # pylint: disable=C0301
         raise MessageBearingError(
             "The CSV header doesn't match what is expected",
             messages=[

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/configuration/v1/__init__.py

@@ -281,7 +281,9 @@ class V1EngineConfig(BaseEngineConfig):
         rules, local_variable_list = [], []
         added_rules: set[RuleName] = set()
 
-        for index, complex_rule_config in enumerate(self.transformations.complex_rules):  # pylint: disable=E1101
+        for index, complex_rule_config in enumerate(
+            self.transformations.complex_rules
+        ):  # pylint: disable=E1101
             rule, local_params, deps = self._resolve_business_rule(complex_rule_config)
             missing_rules = deps - added_rules
             if missing_rules:

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/engine.py

@@ -170,7 +170,9 @@ class CoreEngine(BaseModel):
         exc_value: Optional[Exception],
         traceback: Optional[TracebackType],
     ) -> None:
-        self.main_log.info(f"Exiting pipeline context, clearing {self.cache_prefix!r}")  # pylint: disable=E1101
+        self.main_log.info(
+            f"Exiting pipeline context, clearing {self.cache_prefix!r}"
+        )  # pylint: disable=E1101
         cache_dir = self._cache_dir
         self._cache_dir = None
 
@@ -198,7 +200,9 @@ class CoreEngine(BaseModel):
         """
         output_entities = {}
 
-        self.main_log.info(f"Writing entities to the output location: {self.output_prefix_uri}")  # pylint: disable=E1101
+        self.main_log.info(
+            f"Writing entities to the output location: {self.output_prefix_uri}"
+        )  # pylint: disable=E1101
         for entity_name, entity in entities.items():
             entity = entity.drop(RECORD_INDEX_COLUMN_NAME)
 
@@ -206,9 +210,13 @@ class CoreEngine(BaseModel):
 
             output_uri = joinuri(self.output_prefix_uri, entity_name)
             if get_resource_exists(output_uri):
-                self.main_log.info(f"{output_uri} already exists - will be overwritten")  # pylint: disable=E1101
+                self.main_log.info(
+                    f"{output_uri} already exists - will be overwritten"
+                )  # pylint: disable=E1101
 
-            self.main_log.info(f"+ Writing parquet output to {output_uri!r}")  # pylint: disable=E1101
+            self.main_log.info(
+                f"+ Writing parquet output to {output_uri!r}"
+            )  # pylint: disable=E1101
             entity.write.mode("overwrite").parquet(output_uri)
             spark_session = SparkSession.builder.getOrCreate()
             output_entities[entity_name] = spark_session.read.format("parquet").load(

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/message.py

@@ -36,6 +36,8 @@ class DataContractErrorDetail(BaseModel):
     """Define custom error codes for validation issues raised during the data contract phase"""
 
     error_code: str
+    error_level: Optional[FailureType] = "record"
+    is_informational: Optional[bool] = False
     error_message: Optional[str] = None
     reporting_entity: Optional[str] = None
 
@@ -247,26 +249,14 @@ class FeedbackMessage:  # pylint: disable=too-many-instance-attributes
         messages: Messages = []
         for error_dict in error.errors():
             error_type = error_dict["type"]
+            # TODO - review in pydantic v2 - how handles null vs not provided values
             if "none.not_allowed" in error_type or "value_error.missing" in error_type:
                 category = "Blank"
             else:
                 category = "Bad value"
-            error_code = error_type
-            if "." in error_code:
-                error_code = error_code.split(".", 1)[-1]
-
-            if error_code in INTEGRITY_ERROR_CODES:
-                failure_type: FailureType = "integrity"
-            elif error_code in SUBMISSION_ERROR_CODES:
-                failure_type = "submission"
-            else:
-                failure_type = "record"
 
             error_field = ".".join([idx for idx in error_dict["loc"] if not isinstance(idx, int)])
 
-            is_informational = False
-            if error_code.endswith("warning"):
-                is_informational = True
             error_detail: DataContractErrorDetail = error_details.get(  # type: ignore
                 error_field, DEFAULT_ERROR_DETAIL
             ).get(category)
@@ -276,8 +266,8 @@ class FeedbackMessage:  # pylint: disable=too-many-instance-attributes
                     entity=error_detail.reporting_entity or entity,
                     original_entity=entity,
                     record=record,
-                    failure_type=failure_type,
-                    is_informational=is_informational,
+                    failure_type=error_detail.error_level,  # type: ignore
+                    is_informational=error_detail.is_informational,  # type: ignore
                     error_type=error_type,
                     error_location=error_dict["loc"],  # type: ignore
                     error_message=error_detail.template_message(record, error_dict["loc"]),

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/models.py

@@ -105,6 +105,8 @@ class SubmissionStatisticsRecord(AuditRecord):
 
     record_count: Optional[int]
     """Count of records in the submitted file"""
+    number_submission_rejections: Optional[int]
+    """Number of submission rejections raised following validation"""
     number_record_rejections: Optional[int]
     """Number of record rejections raised following validation"""
     number_warnings: Optional[int]

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/metadata_parser/__init__.py

@@ -1,2 +1,2 @@
 """Module for parsing out the DVE config and generating models,
-   includes domain types for validation"""
+includes domain types for validation"""

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/pipeline/pipeline.py

@@ -1,5 +1,6 @@
 # pylint: disable=protected-access,too-many-instance-attributes,too-many-arguments,line-too-long
 """Generic Pipeline object to define how DVE should be interacted with."""
+
 import json
 import logging
 import re
@@ -42,6 +43,7 @@ from dve.parser import file_handling as fh
 from dve.parser.file_handling.implementations.file import LocalFilesystemImplementation
 from dve.parser.file_handling.service import _get_implementation
 from dve.pipeline.utils import SubmissionStatus, deadletter_file, load_config, load_reader
+from dve.reporting.constants import ErrorReportCategories
 from dve.reporting.error_report import ERROR_SCHEMA, calculate_aggregates
 
 PERMISSIBLE_EXCEPTIONS: tuple[type[Exception]] = (
@@ -223,7 +225,7 @@ class BaseDVEPipeline:
                             submission_file_uri,
                             model_name,
                             stringify_model(model),  # type: ignore
-                            get_all_model_fields(models.values())  # type: ignore
+                            get_all_model_fields(models.values()),  # type: ignore
                         ),
                         f"{out}{model_name}",
                     )
@@ -379,9 +381,6 @@ class BaseDVEPipeline:
                     failed.append((submission_info, submission_status))
                 else:
                     success.append((submission_info, submission_status))
-            except AttributeError as exc:
-                self._logger.error(f"File transformation raised exception: {exc}")
-                raise exc
             except PERMISSIBLE_EXCEPTIONS as exc:
                 self._logger.warning(
                     f"File transformation raised exception: {exc}. Will be retried later."
@@ -509,9 +508,6 @@ class BaseDVEPipeline:
                 submission_info: SubmissionInfo
                 submission_status: SubmissionStatus
                 submission_info, submission_status = future.result()
-            except AttributeError as exc:
-                self._logger.error(f"Data Contract raised exception: {exc}")
-                raise exc
             except PERMISSIBLE_EXCEPTIONS as exc:
                 self._logger.warning(
                     f"Data Contract raised exception: {exc}. Will be retried later."
@@ -616,8 +612,19 @@ class BaseDVEPipeline:
             submission_status.processing_failed = True
 
         for entity_name, entity in entity_manager.entities.items():
+            # Note BI filtering done within the apply_rules
+            self._logger.info(f"applying data contract filter to {entity_name}.")
+            if not entity_name.startswith("Original"):
+                filtered_entity = self._step_implementations.filter_data_contract_record_rejections(
+                    working_directory,
+                    entity,
+                    entity_name,
+                )
+            else:
+                self._logger.info(f"Skipping {entity_name}. Marked original.")
+                filtered_entity = entity
             projected = self._step_implementations.write_parquet(  # type: ignore
-                entity,
+                filtered_entity,
                 fh.joinuri(
                     self.processed_files_path,
                     submission_info.submission_id,
@@ -630,11 +637,17 @@ class BaseDVEPipeline:
             )
 
         submission_status.number_of_records = self.get_entity_count(
-            entity=entity_manager.entities[
-                f"""Original{rules.global_variables.get(
+            entity=entity_manager.entities[f"""Original{rules.global_variables.get(
                                               'entity',
-                                              submission_info.dataset_id)}"""
-            ]
+                                              submission_info.dataset_id)}"""]
+        )
+        submission_status.number_of_records_rejected = (
+            submission_status.number_of_records
+            - self.get_entity_count(
+                entity_manager.entities[
+                    rules.global_variables.get("entity", submission_info.dataset_id)
+                ]
+            )
         )
 
         return submission_info, submission_status
@@ -682,9 +695,6 @@ class BaseDVEPipeline:
                     unsucessful_files.append((submission_info, submission_status))  # type: ignore
                 else:
                     successful_files.append((submission_info, submission_status))  # type: ignore
-            except AttributeError as exc:
-                self._logger.error(f"Business Rules raised exception: {exc}")
-                raise exc
             except PERMISSIBLE_EXCEPTIONS as exc:
                 self._logger.warning(
                     f"Business Rules raised exception: {exc}. Will be retried later."
@@ -758,10 +768,12 @@ class BaseDVEPipeline:
 
                 df = pl.DataFrame(errors, schema={key: pl.Utf8() for key in errors[0]})  # type: ignore
                 df = df.with_columns(
-                    pl.when(pl.col("Status") == pl.lit("error"))  # type: ignore
-                    .then(pl.lit("Submission Failure"))  # type: ignore
-                    .otherwise(pl.lit("Warning"))  # type: ignore
-                    .alias("error_type")
+                    pl.when(pl.col("Status") == pl.lit("informational"))
+                    .then(pl.lit("Warning"))
+                    .when(pl.col("FailureType") == pl.lit("submission"))  # type: ignore
+                    .then(pl.lit(ErrorReportCategories.FILE_REJECTION.reporting_name))  # type: ignore
+                    .otherwise(pl.lit(ErrorReportCategories.RECORD_REJECTION.reporting_name))  # type: ignore
+                    .alias("error_type")  # type: ignore
                 )
                 df = df.select(
                     pl.col("Entity").alias("Table"),  # type: ignore
@@ -823,8 +835,13 @@ class BaseDVEPipeline:
             sub_stats = SubmissionStatisticsRecord(
                 submission_id=submission_info.submission_id,
                 record_count=submission_status.number_of_records,
-                number_record_rejections=err_types.get("Submission Failure", 0),
-                number_warnings=err_types.get("Warning", 0),
+                number_submission_rejections=err_types.get(
+                    ErrorReportCategories.FILE_REJECTION.reporting_name, 0
+                ),
+                number_record_rejections=err_types.get(
+                    ErrorReportCategories.RECORD_REJECTION.reporting_name, 0
+                ),
+                number_warnings=err_types.get(ErrorReportCategories.WARNING.reporting_name, 0),
             )
 
         summary_dict = {
@@ -835,7 +852,7 @@ class BaseDVEPipeline:
         summary_items = er.SummaryItems(
             submission_status=submission_status,
             summary_dict=summary_dict,
-            row_headings=["Submission Failure", "Warning"],
+            row_headings=[e.reporting_name for e in ErrorReportCategories],
         )
 
         workbook = er.ExcelFormat(
@@ -894,9 +911,6 @@ class BaseDVEPipeline:
             try:
                 submission_info, submission_status, submission_stats, feedback_uri = future.result()
                 reports.append((submission_info, submission_status, submission_stats, feedback_uri))
-            except AttributeError as exc:
-                self._logger.error(f"Error reports raised exception: {exc}")
-                raise exc
             except PERMISSIBLE_EXCEPTIONS as exc:
                 self._logger.warning(
                     f"Error reports raised exception: {exc}. Will be retried later."

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/pipeline/utils.py

@@ -86,10 +86,12 @@ class SubmissionStatus:
         self,
         validation_failed: bool = False,
         number_of_records: Optional[int] = None,
+        number_of_records_rejected: Optional[int] = None,
         processing_failed: bool = False,
     ):
         self.validation_failed = validation_failed
         self.number_of_records = number_of_records
+        self.number_of_records_rejected = number_of_records_rejected
         self.processing_failed = processing_failed
 
     @property

data_validation_engine-0.8.2/src/dve/reporting/constants.py

@@ -0,0 +1,34 @@
+"""
+Constants used within the error reports
+"""
+
+from enum import Enum
+
+
+class ErrorReportCategories(Enum):
+    """
+    Categories available within the error report.
+    """
+
+    FILE_REJECTION = 1, "File Rejection"
+    RECORD_REJECTION = 2, "Record Rejection"
+    WARNING = 3, "Warning"
+
+    @property
+    def reporting_name(self):
+        """
+        The error report 'friendly' name.
+        """
+        return self.value[1]
+
+
+class ErrorReportStatus:  # pylint: disable=R0903
+    """
+    Statuses available within the error report.
+    """
+
+    PROCESSING_FAILED = "There was an issue processing the submission. Please contact support."
+    FILE_REJECTION = "File has been rejected"
+    RECORD_REJECTION = "File has been accepted with record rejections"
+    ACCEPTED_WITH_WARNING = "File has been accepted, all records accepted with warnings"
+    ACCEPTED = "File has been accepted, no issues to report"

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/reporting/error_report.py

@@ -11,6 +11,7 @@ from polars import DataFrame, LazyFrame, Utf8, col  # type: ignore
 from dve.common.error_utils import conditional_cast
 from dve.core_engine.message import FeedbackMessage
 from dve.parser.file_handling.service import open_stream
+from dve.reporting.constants import ErrorReportCategories
 
 ERROR_SCHEMA = {
     "Table": Utf8(),
@@ -85,7 +86,7 @@ def create_error_dataframe(errors: deque[FeedbackMessage], key_fields):
 
     df = df.with_columns(  # type: ignore
         pl.when(pl.col("Status") == pl.lit("error"))  # type: ignore
-        .then(pl.lit("Submission Failure"))  # type: ignore
+        .then(pl.lit(ErrorReportCategories.FILE_REJECTION.reporting_name))  # type: ignore
         .otherwise(pl.lit("Warning"))  # type: ignore
         .alias("error_type")
     )

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/reporting/excel_report.py

@@ -17,6 +17,7 @@ from polars import DataFrame
 from polars.exceptions import ColumnNotFoundError
 
 from dve.pipeline.utils import SubmissionStatus
+from dve.reporting.constants import ErrorReportCategories, ErrorReportStatus
 
 
 @dataclass
@@ -91,18 +92,18 @@ class SummaryItems:
     def get_submission_status(self, aggregates: DataFrame) -> str:
         """Returns the status of the submission based on the error data"""
         if self.submission_status.processing_failed:
-            return "There was an issue processing the submission. Please contact support."
-        if self.submission_status.validation_failed:
-            return "File has been rejected"
+            return ErrorReportStatus.PROCESSING_FAILED
         if aggregates.is_empty():
-            return "File has been accepted, no issues to report"
+            return ErrorReportStatus.ACCEPTED
         failures = aggregates["Type"].unique()
-        if "Submission Failure" in failures:
-            status = "File has been rejected"
-        elif "Warning" in failures:
-            status = "File has been accepted, all records accepted with warnings"
+        if ErrorReportCategories.FILE_REJECTION.reporting_name in failures:
+            status = ErrorReportStatus.FILE_REJECTION
+        elif ErrorReportCategories.RECORD_REJECTION.reporting_name in failures:
+            status = ErrorReportStatus.RECORD_REJECTION
+        elif ErrorReportCategories.WARNING.reporting_name in failures:
+            status = ErrorReportStatus.ACCEPTED_WITH_WARNING
         else:
-            status = "File has been accepted, no issues to report"
+            status = ErrorReportStatus.ACCEPTED
         return status
 
     def _write_table(
@@ -141,6 +142,28 @@ class SummaryItems:
         for key, value in self.summary_dict.items():
             summary.append(["", _key_renames.get(key, key), str(value)])
 
+        summary.append(
+            [
+                "",
+                "Total Number of Records Processed",
+                (
+                    self.submission_status.number_of_records
+                    if self.submission_status.number_of_records
+                    else 0
+                ),  # pylint: disable=C0301
+            ]
+        )
+        if status not in (
+            ErrorReportStatus.PROCESSING_FAILED,
+            ErrorReportStatus.FILE_REJECTION,
+        ):
+            summary.append(
+                [
+                    "",
+                    "Total Number of Records Rejected",
+                    self.submission_status.number_of_records_rejected,
+                ]
+            )
         summary.append(["", ""])
 
 

{data_validation_engine-0.8.1 → data_validation_engine-0.8.2}/src/dve/core_engine/backends/implementations/spark/readers/csv.py

@@ -9,13 +9,13 @@ from pyspark.sql import DataFrame, SparkSession
 from pyspark.sql.types import StructType
 
 from dve.core_engine.backends.base.reader import read_function
-from dve.core_engine.backends.readers.csv import CSVFileReader
 from dve.core_engine.backends.exceptions import EmptyFileError
 from dve.core_engine.backends.implementations.spark.spark_helpers import (
     get_type_from_annotation,
     spark_record_index,
     spark_write_parquet,
 )
+from dve.core_engine.backends.readers.csv import CSVFileReader
 from dve.core_engine.type_hints import URI, EntityName
 from dve.parser.file_handling import get_content_length