data-validation-engine 0.9.0__tar.gz → 0.10.0__tar.gz

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/PKG-INFO

@@ -1,14 +1,14 @@
 Metadata-Version: 2.4
 Name: data-validation-engine
-Version: 0.9.0
+Version: 0.10.0
 Summary: `nhs data validation engine` is a framework used to validate data
 License-Expression: MIT
 License-File: LICENSE
 Author: NHS England
 Author-email: england.contactus@nhs.net
-Requires-Python: >=3.10,<3.13
+Requires-Python: >=3.11,<3.13
+Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Operating System :: POSIX :: Linux
@@ -18,7 +18,7 @@ Requires-Dist: Jinja2 (==3.1.6)
 Requires-Dist: boto3 (>=1.34.162,<1.36)
 Requires-Dist: botocore (>=1.34.162,<1.36)
 Requires-Dist: delta-spark (>=3.0.0,<=3.2.0)
-Requires-Dist: duckdb (==1.1.3)
+Requires-Dist: duckdb (>=1.4,<1.4.5)
 Requires-Dist: lxml (==6.1.1)
 Requires-Dist: numpy (==1.26.4)
 Requires-Dist: openpyxl (==3.1.5)
@@ -27,8 +27,8 @@ Requires-Dist: polars (==0.20.31)
 Requires-Dist: pyarrow (==23.0.1)
 Requires-Dist: pydantic (==2.13.4)
 Requires-Dist: pyspark (>=3.5.0,<=3.5.5)
-Requires-Dist: typing_extensions (==4.15.0)
-Requires-Dist: urllib3 (==2.7.0)
+Requires-Dist: typing_extensions (==4.16.0)
+Requires-Dist: urllib3 (==2.8.0)
 Project-URL: Changelog, https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md
 Project-URL: Documentation, https://nhsdigital.github.io/data-validation-engine/
 Project-URL: Homepage, https://github.com/NHSDigital/data-validation-engine
@@ -55,6 +55,7 @@ __The DVE offers__:
 - Format normalization to Parquet for a unified data representation
 - Data modelling and typecasting
 - Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Validate and enforce referential integrity checks with minimal configuration
 - Deriving new fields and entities
 - Clear validation reporting, including summary insights and record-level error messages
 
@@ -79,7 +80,8 @@ Below is a list of features that we would like to implement or have been request
 | Uplift to Python 3.11                                                           | 0.2.0             | Yes       |
 | Uplift Pyspark to 3.5                                                           | 0.8.0             | Yes       |
 | Allow DVE to run on Python 3.12+                                                | 0.8.0             | Yes       |
-| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes        |
+| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes       |
+| Upgrade DuckDB to v1.4                                                          | 0.10.0            | Yes       |
 | Uplift Pyspark to 4.0+                                                          | TBA               | No        |
 | Polars upgrade to v1+ | TBA | No |
 | DuckDB upgrade to v1.5+ | TBA | No |

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/README.md

@@ -17,6 +17,7 @@ __The DVE offers__:
 - Format normalization to Parquet for a unified data representation
 - Data modelling and typecasting
 - Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Validate and enforce referential integrity checks with minimal configuration
 - Deriving new fields and entities
 - Clear validation reporting, including summary insights and record-level error messages
 
@@ -41,7 +42,8 @@ Below is a list of features that we would like to implement or have been request
 | Uplift to Python 3.11                                                           | 0.2.0             | Yes       |
 | Uplift Pyspark to 3.5                                                           | 0.8.0             | Yes       |
 | Allow DVE to run on Python 3.12+                                                | 0.8.0             | Yes       |
-| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes        |
+| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes       |
+| Upgrade DuckDB to v1.4                                                          | 0.10.0            | Yes       |
 | Uplift Pyspark to 4.0+                                                          | TBA               | No        |
 | Polars upgrade to v1+ | TBA | No |
 | DuckDB upgrade to v1.5+ | TBA | No |

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/pyproject.toml

@@ -7,8 +7,8 @@ authors = [
 ]
 readme = "README.md"
 classifiers = [
+    "Development Status :: 4 - Beta",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Operating System :: POSIX :: Linux",
@@ -25,17 +25,17 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
 Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
 
 [tool.poetry]
-version = "0.9.0"
+version = "0.10.0"
 packages = [
     { include = "dve", from = "src" },
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.10,<3.13"  # breaking changes beyond 3.12
+python = ">=3.11,<3.13"  # breaking changes beyond 3.12
 boto3 = ">=1.34.162,<1.36"  # breaking change beyond 1.36
 botocore = ">=1.34.162,<1.36"  # breaking change beyond 1.36
 delta-spark = ">=3.0.0,<=3.2.0"
-duckdb = "1.1.3"  # breaking changes beyond 1.1
+duckdb = ">=1.4,<1.4.5"
 Jinja2 = "3.1.6"
 lxml = "6.1.1"
 numpy = "1.26.4"
@@ -45,8 +45,8 @@ polars = "0.20.31"
 pyarrow = "23.0.1"
 pydantic = "2.13.4"
 pyspark = ">=3.5.0,<=3.5.5"
-typing_extensions = "4.15.0"
-urllib3 = "2.7.0"  # dependency of boto3 & botocore
+typing_extensions = "4.16.0"
+urllib3 = "2.8.0"  # dependency of boto3 & botocore
 
 [tool.poetry.group.dev]
 optional = true
@@ -59,8 +59,11 @@ include-groups = [
 commitizen = "4.9.1"
 pre-commit = "4.3.0"
 charset-normalizer = "3.4.6"
-python-discovery = "1.2.0"
+python-discovery = "1.6.1"
 idna = "3.15"  # dependency of requests
+virtualenv = "21.14.9"
+certifi = "2026.10.23"
+filelock = "4.2.1"
 
 [tool.poetry.group.test]
 optional = true
@@ -71,7 +74,7 @@ coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.2.14"}
 cryptography = "50.0.0"  # dependency of `moto`
 requests = "2.33.0"  # dependency of `moto`
-Werkzeug = "3.1.6"
+Werkzeug = "3.1.9"
 pytest = "9.0.3"
 pytest-lazy-fixtures = "1.4.0"  # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
 xlsx2csv = "0.8.4"  # polars requirement
@@ -80,7 +83,7 @@ xlsx2csv = "0.8.4"  # polars requirement
 optional = true
 
 [tool.poetry.group.lint.dependencies]
-black = "26.3.1"
+black = "26.10.1"
 astroid = "3.3.9"
 isort = "5.13.2"
 pylint = "3.3.9"
@@ -108,7 +111,7 @@ mkdocs = "1.6.1"
 mkdocstrings = { version = "1.0.3", extras = ["python"] }
 griffelib = "2.0.1"
 pymdown-extensions = "11.0.1"
-zensical = "0.0.46"
+zensical = "0.0.63"
 
 [tool.ruff]
 line-length = 100

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/src/dve/common/error_utils.py

@@ -5,7 +5,7 @@ import json
 import logging
 from collections.abc import Iterable
 from itertools import chain
-from multiprocessing import Queue
+from queue import Queue
 from threading import Thread
 from typing import Optional, Union
 

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/src/dve/core_engine/backends/base/contract.py

@@ -443,10 +443,6 @@ class BaseDataContract(Generic[EntityType], ABC):
                 ],
             )
 
-        if contract_metadata.cache_originals:
-            for entity_name in list(entities):
-                entities[f"Original{entity_name}"] = entities[entity_name]
-
         return entities, feedback_errors_uri, successful, processing_errors_uri
 
     def read_parquet(self, path: URI, **kwargs) -> EntityType:

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/src/dve/core_engine/backends/base/reader.py

@@ -8,8 +8,16 @@ from typing import Any, ClassVar, Optional, TypeVar
 from pydantic import BaseModel
 from typing_extensions import Protocol
 
-from dve.core_engine.backends.exceptions import MessageBearingError, ReaderLacksEntityTypeSupport
+from dve.core_engine.backends.exceptions import (
+    CriticalMessageBearingError,
+    MessageBearingError,
+    ReaderLacksEntityTypeSupport,
+)
 from dve.core_engine.backends.types import EntityName, EntityType
+from dve.core_engine.configuration.v1 import (
+    AllowedAdditionalReaderChecks,
+    _ReaderAdditionalChecksConfig,
+)
 from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, ArbitraryFunction, WrapDecorator
 from dve.parser.file_handling.service import open_stream
@@ -65,6 +73,10 @@ class BaseFileReader(ABC):
     decorated with the '@read_function' decorator, and is used in `read_entity_type`.
 
     """
+    ft_error_code: Optional[str] = "MalformedFile"
+    """Default error code for when a file/submission cannot be parsed succesfully."""
+    ft_error_message: Optional[str] = "The resource doesn't seem to be a valid text file"
+    """Default error message for when a file/submission cannot be parsed succesfully."""
 
     def __init_subclass__(cls, *_, **__) -> None:
         """When this class is subclassed, create and populate the `__read_methods__`
@@ -109,7 +121,10 @@ class BaseFileReader(ABC):
         entity_name: EntityName,
         schema: type[BaseModel],
         all_model_fields: Optional[set[str]] = None,
-    ) -> EntityType:
+        additional_checks: Optional[
+            dict[AllowedAdditionalReaderChecks, _ReaderAdditionalChecksConfig]
+        ] = None,
+    ):
         """Read to the specified entity type, if supported.
 
         NOTE: Simple types should either be returned as strings (if present) or
@@ -117,21 +132,47 @@ class BaseFileReader(ABC):
         data contract.
 
         """
-        if entity_name == Iterator[dict[str, Any]]:
-            return self.read_to_py_iterator(
+        additional_checks = additional_checks or {}
+
+        self.raise_if_not_sensible_file(resource, entity_name)
+
+        if entity_type == Iterator[dict[str, Any]]:
+            entity = self.read_to_py_iterator(
                 resource, entity_name, schema, all_model_fields  # type: ignore
             )
 
-        self.raise_if_not_sensible_file(resource, entity_name)
+        else:
 
-        try:
-            reader_func = self.__read_methods__[entity_type]
-        except KeyError as err:
-            raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
+            try:
+                reader_func = self.__read_methods__[entity_type]
+            except KeyError as err:
+                raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
 
-        return reader_func(
-            self, resource, entity_name, schema, all_model_fields=all_model_fields  # type: ignore
-        )
+            entity = reader_func(
+                self,
+                resource,
+                entity_name,
+                schema,
+                all_model_fields=all_model_fields,  # type: ignore
+            )
+
+        if config := additional_checks.get("check_empty"):
+            if self.check_entity_empty(entity):
+                raise MessageBearingError(
+                    f"The mandatory entity {entity_name} is empty",
+                    messages=[
+                        FeedbackMessage(
+                            entity=entity_name,
+                            record=None,
+                            failure_type="submission",
+                            error_location=entity_name,
+                            error_code=config.error_code,
+                            error_message=config.error_message,
+                        )
+                    ],
+                )
+
+        return entity
 
     def add_record_index(self, entity: EntityType, **kwargs) -> EntityType:
         """Add a record index to the entity"""
@@ -141,6 +182,10 @@ class BaseFileReader(ABC):
         """Drop a record index to the entity"""
         raise NotImplementedError(f"drop_record_index not implemented in {self.__class__}")
 
+    def check_entity_empty(self, entity: EntityType) -> bool:
+        """Determine if the entity supplied is empty"""
+        raise NotImplementedError(f"check_entity_empty not implemented in {self.__class__}")
+
     def write_parquet(
         self,
         entity: EntityType,
@@ -171,20 +216,22 @@ class BaseFileReader(ABC):
             return False
         return True
 
-    def raise_if_not_sensible_file(self, resource: URI, entity_name: str):
+    def raise_if_not_sensible_file(
+        self,
+        resource: URI,
+        entity_name: str,
+    ):
         """Sense check that the file is a text file. Raise error if doesn't
         appear to be the case."""
         if not self._check_likely_text_file(resource):
-            raise MessageBearingError(
+            raise CriticalMessageBearingError(
                 "The submitted file doesn't appear to be text",
-                messages=[
-                    FeedbackMessage(
-                        entity=entity_name,
-                        record=None,
-                        failure_type="submission",
-                        error_location="Whole File",
-                        error_code="MalformedFile",
-                        error_message="The resource doesn't seem to be a valid text file",
-                    )
-                ],
+                message=FeedbackMessage(
+                    entity=entity_name,
+                    record=None,
+                    failure_type="submission",
+                    error_location="Whole File",
+                    error_code=self.ft_error_code,
+                    error_message=self.ft_error_message,
+                ),
             )

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/src/dve/core_engine/backends/base/rules.py

@@ -3,8 +3,8 @@
 import logging
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from collections.abc import Iterable
-from typing import Any, ClassVar, Generic, NoReturn, Optional, TypeVar
+from collections.abc import Iterable, Iterator
+from typing import Any, ClassVar, Generic, MutableMapping, NoReturn, Optional, TypeVar
 from uuid import uuid4
 
 from typing_extensions import Literal, Protocol, get_type_hints
@@ -27,6 +27,7 @@ from dve.core_engine.backends.metadata.rules import (
     CopyEntity,
     DeferredFilter,
     EntityRemoval,
+    GroupIdentification,
     HeaderJoin,
     ImmediateFilter,
     InnerJoin,
@@ -43,8 +44,11 @@ from dve.core_engine.backends.metadata.rules import (
     TableUnion,
 )
 from dve.core_engine.backends.types import Entities, EntityType, StageSuccessful
+from dve.core_engine.configuration.v1.hierarchy import EntityHierarchy, HierarchyNode
 from dve.core_engine.exceptions import CriticalProcessingError
 from dve.core_engine.loggers import get_logger
+from dve.core_engine.message import FeedbackMessage
+from dve.core_engine.templating import template_object
 from dve.core_engine.type_hints import URI, DVEStageName, EntityName, Messages, TemplateVariables
 
 T_contra = TypeVar("T_contra", bound=AbstractStep, contravariant=True)
@@ -55,6 +59,8 @@ _StepFunctions = dict[type[T], "_UnboundStepFunction[T]"]
 """A convenience type indicating a mapping from config type to step method."""
 Stage = Literal["Pre-filter", "Filter", "Post-filter"]
 """The name of a stage within a rule."""
+TempTableName = str
+"""temp tables to cache intermediate results"""
 
 
 class _UnboundStepFunction(Generic[T_contra], Protocol):  # pylint: disable=too-few-public-methods
@@ -128,6 +134,7 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
     ):
         self.logger = logger or get_logger(type(self).__name__)
         """The `logging.Logger instance for the data contract config."""
+        self.entity_cache_tracker: MutableMapping[EntityName, TempTableName] = {}
 
     @classmethod
     @abstractmethod
@@ -307,7 +314,8 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
         """
         raise NotImplementedError
 
-    def identify_orphans(self, entities: Entities, *, config: OrphanIdentification) -> Messages:
+    @abstractmethod
+    def identify_orphans(self, entities: Entities, *, config: OrphanIdentification) -> Iterable:
         """Identify records in an entity which don't have at least one corresponding
         match in the target. A new boolean column will be added to `entity` ('IsOrphaned')
         indicating whether the condition matched.
@@ -320,6 +328,14 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def check_mandatory_group(self, entities: Entities, *, config: GroupIdentification) -> Iterator:
+        """
+        Check that a mandatory key in an entity has at least one valid entry in the all the child
+        entities.
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def union(self, entities: Entities, *, config: TableUnion) -> Messages:
         """Union two entities together, taking the columns from each by name.
@@ -352,6 +368,157 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
 
         """
 
+    def identify_and_remove_orphans(
+        self,
+        working_directory: URI,
+        entities: Entities,
+        entity_hierarchy: EntityHierarchy,
+        key_fields: Optional[dict[str, list[str]]] = None,
+    ) -> tuple[Messages, dict[EntityName, bool]]:
+        """
+        Identifies and removes orphan records by traversing the EntityHierarchy object.
+        An orphan is a child record whose parent FK does not exist in the parent entity.
+        Processes recursively: removes orphans at each level, then processes children.
+        """
+
+        def process_node(node: HierarchyNode) -> bool:
+            """Identify orphans and remove in a given node"""
+            issues_found: bool = False
+            if node.parent_entity is None:
+                return issues_found
+
+            self.logger.info(f"Checking for orphan records in {node.entity_name}")
+
+            join_expr = " AND ".join(
+                f"{node.parent_entity}.{k} = {node.entity_name}.{v}"
+                for k, v in node.join_fields.items()
+            )
+            location = list(node.join_fields.values())[0]
+            with BackgroundMessageWriter(
+                working_directory=working_directory,
+                dve_stage=self.__stage_name__,
+                key_fields=key_fields,
+                logger=self.logger,
+            ) as msg_writer:
+                _orph_records = self.identify_orphans(
+                    entities=entities,
+                    config=OrphanIdentification(
+                        id=list(node.join_fields.values())[0],
+                        entity_name=node.entity_name,
+                        target_name=node.parent_entity,
+                        join_condition=join_expr,
+                    ),
+                )
+                _messages = [
+                    FeedbackMessage(
+                        entity=node.entity_name,
+                        record=record,  # type: ignore
+                        error_location=location,
+                        error_message=template_object(node.missing_parent_id_error_message, record),
+                        failure_type="record",
+                        error_type="record",
+                        error_code=node.missing_parent_id_error_code,
+                        reporting_field=location,
+                        category="Parent Missing",
+                    )
+                    for record in _orph_records
+                ]
+                msg_writer.write_queue.put(_messages)
+
+            self.cache_entity(node.entity_name, entities)
+
+            _orph_count = len(_messages)
+
+            self.logger.info(
+                f"Found {_orph_count} orphan records between {node.entity_name} and {node.parent_entity}"  # pylint: disable=C0301
+            )
+
+            return _orph_count > 0
+
+        entity_issues_found: dict[EntityName, bool] = {}
+
+        for tree in entity_hierarchy.entity_trees.values():
+            for node in tree.iterate_root_down():
+                entity_issues_found[node.entity_name] = process_node(node)
+
+        return [], entity_issues_found
+
+    def identify_and_remove_missing_mandatory_groups(
+        self,
+        working_directory: URI,
+        entities: Entities,
+        entity_hierarchy: EntityHierarchy,
+        key_fields: Optional[dict[str, list[str]]] = None,
+    ) -> tuple[Messages, dict[EntityName, bool]]:
+        """
+        Identify that an entity with a mandatory key has at least one valid child record.
+        """
+
+        def process_node(node: HierarchyNode) -> bool:
+            """Identify at least one valid child for a mandatory entity at a given node."""
+            if node.parent_entity is None or not node.mandatory:
+                return False
+
+            self.logger.info(
+                f"Identifying that mandatory entity `{node.parent_entity}` has at least 1 valid child record in {node.entity_name}"  # pylint: disable=C0301
+            )
+
+            join_expr = " AND ".join(
+                f"{node.parent_entity}.{k} = {node.entity_name}.{v}"
+                for k, v in node.join_fields.items()
+            )
+
+            with BackgroundMessageWriter(
+                working_directory=working_directory,
+                dve_stage=self.__stage_name__,
+                key_fields=key_fields,
+                logger=self.logger,
+            ) as msg_writer:
+                location = next(iter(node.join_fields.values()))
+                missing_children_records = self.check_mandatory_group(
+                    entities=entities,
+                    config=GroupIdentification(
+                        entity_name=node.parent_entity,
+                        target_name=node.entity_name,
+                        join_condition=join_expr,
+                    ),
+                )
+                _messages = [
+                    FeedbackMessage(
+                        entity=node.parent_entity,
+                        record=record,  # type: ignore
+                        error_location=location,
+                        error_message=template_object(node.no_valid_records_error_message, record),
+                        failure_type="record",
+                        error_type="record",
+                        error_code=node.no_valid_records_error_code,
+                        reporting_field=location,
+                        category="Children missing",
+                    )
+                    for record in missing_children_records
+                ]
+                msg_writer.write_queue.put(_messages)
+            self.cache_entity(node.parent_entity, entities)
+
+            _no_valid_child_records: int = len(_messages)
+
+            self.logger.info(
+                f"Found {_no_valid_child_records} records with no valid children in {node.parent_entity}."  # pylint: disable=C0301
+            )
+
+            return _no_valid_child_records > 0
+
+        entity_issues_found: dict[EntityName, bool] = {}
+
+        for tree in entity_hierarchy.entity_trees.values():
+            for node in tree.iterate_lowest_descendent_up():
+                if node.parent_entity and node.mandatory:
+                    result = process_node(node)
+                    if result:
+                        entity_issues_found[node.parent_entity] = result
+
+        return [], entity_issues_found
+
     # pylint: disable=R0912,R0914
     def apply_sync_filters(
         self,
@@ -428,6 +595,7 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
                                 excluded_columns=filter_column_names,
                                 reporting=rule.reporting,
                                 parent=rule.parent,
+                                error_on_null=True,
                             ),
                         )
                         if not success:
@@ -459,6 +627,7 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
                                 expression=f"NOT ({rule.expression})",
                                 reporting=rule.reporting,
                                 parent=rule.parent,
+                                error_on_null=True,
                             ),
                         )
                         if not success:
@@ -691,3 +860,25 @@ class BaseStepImplementations(Generic[EntityType], ABC):  # pylint: disable=too-
     ):
         """Method to filter out record rejection errors from the data contract for a given entity"""
         raise NotImplementedError()
+
+    @staticmethod
+    def get_entity_count(entity: EntityType) -> int:
+        """Method to get count of records in entity"""
+        raise NotImplementedError()
+
+    def cache_entity(self, entity_name: EntityName, entities: Entities):
+        """Store the materialised query in memory and update entity to query directly.
+        If the entity is already cached, the new cache should be created first, then the old one
+        removed as part of the function (in case the newer cache depends on the older one)."""
+        raise NotImplementedError()
+
+    def _remove_cached_artifact(self, entity_name: EntityName):
+        """Delete artifact in memory and clear from the entity cache keeping track.
+        This should not be used directly as removing artifacts from memory, may lead to some
+        entities being unable to be processed as their execution plans depend on these artifacts."""
+        raise NotImplementedError()
+
+    def clear_entity_cache(self):
+        """Helper method to remove all artifacts and cache trackers at end of processing."""
+        for entity_name in list(self.entity_cache_tracker):
+            self._remove_cached_artifact(entity_name)

{data_validation_engine-0.9.0 → data_validation_engine-0.10.0}/src/dve/core_engine/backends/exceptions.py

@@ -33,27 +33,39 @@ class MessageBearingError(BackendError):
         """The messages to be returned as part of the error."""
 
 
-class UnableToParseCSVError(MessageBearingError):
+class CriticalMessageBearingError(BackendError):
+    """
+    A backend error that comes with a pre-created message.
+    The intention of this exception vs MessageBearingError is that
+    this should be used to halt the processing of a submission.
+    """
+
+    def __init__(self, *args: object, message: FeedbackMessage) -> None:
+        super().__init__(*args)
+        self.message = message
+        """The message to be returned as part of the error."""
+
+
+class UnableToParseCSVError(CriticalMessageBearingError):
     """An error raised when unable to parse a CSV file"""
 
     def __init__(
-        self, entity_name: str, field_check_error_message: str, field_check_error_code: str
+        self,
+        entity_name: Optional[str],
+        error_message: Optional[str],
+        error_code: Optional[str],
     ):
         super().__init__(
-            messages=[
-                FeedbackMessage(
-                    entity="csv_structure",
-                    record={
-                        entity_name: "Unable to parse file. Please check the structure of the file."
-                    },
-                    failure_type="submission",
-                    is_informational=False,
-                    error_type="csv read",
-                    error_location=entity_name,
-                    error_message=field_check_error_message,
-                    error_code=field_check_error_code,
-                )
-            ]
+            message=FeedbackMessage(
+                entity=entity_name,
+                record=None,
+                failure_type="submission",
+                is_informational=False,
+                error_type="csv read",
+                error_message=error_message
+                or "Unable to parse the CSV file. Please check the structure of your CSV.",  # pylint: disable=C0301
+                error_code=error_code or "MalformedCSV",
+            )
         )