data-validation-engine 0.8.5__tar.gz → 0.10.0__tar.gz

data_validation_engine-0.10.0/PKG-INFO

@@ -0,0 +1,100 @@
+Metadata-Version: 2.4
+Name: data-validation-engine
+Version: 0.10.0
+Summary: `nhs data validation engine` is a framework used to validate data
+License-Expression: MIT
+License-File: LICENSE
+Author: NHS England
+Author-email: england.contactus@nhs.net
+Requires-Python: >=3.11,<3.13
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Typing :: Typed
+Requires-Dist: Jinja2 (==3.1.6)
+Requires-Dist: boto3 (>=1.34.162,<1.36)
+Requires-Dist: botocore (>=1.34.162,<1.36)
+Requires-Dist: delta-spark (>=3.0.0,<=3.2.0)
+Requires-Dist: duckdb (>=1.4,<1.4.5)
+Requires-Dist: lxml (==6.1.1)
+Requires-Dist: numpy (==1.26.4)
+Requires-Dist: openpyxl (==3.1.5)
+Requires-Dist: pandas (==2.3.3)
+Requires-Dist: polars (==0.20.31)
+Requires-Dist: pyarrow (==23.0.1)
+Requires-Dist: pydantic (==2.13.4)
+Requires-Dist: pyspark (>=3.5.0,<=3.5.5)
+Requires-Dist: typing_extensions (==4.16.0)
+Requires-Dist: urllib3 (==2.8.0)
+Project-URL: Changelog, https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://nhsdigital.github.io/data-validation-engine/
+Project-URL: Homepage, https://github.com/NHSDigital/data-validation-engine
+Project-URL: Issues, https://github.com/NHSDigital/data-validation-engine/issues
+Project-URL: Repository, https://github.com/NHSDigital/data-validation-engine.git
+Description-Content-Type: text/markdown
+
+<h1 style="display: flex; align-items: center; gap: 10px;">
+    <img src="https://raw.githubusercontent.com/NHSDigital/data-validation-engine/616b55890306db4546177f7effac48ca241857ec/overrides/.icons/nhseng.svg" alt="" width="5%" height="100%" align="left">
+    Data Validation Engine
+</h1>
+
+![License](https://img.shields.io/github/license/NHSDigital/data-validation-engine)
+![PyPi](https://img.shields.io/pypi/v/data-validation-engine)
+![Conda](https://anaconda.org/nhs/data-validation-engine/badges/version.svg)
+[![CI Unit Tests](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml)
+[![CI Formatting & Linting](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml)
+
+The Data Validation Engine (DVE) is a configuration-driven data validation library created and used by NHS England. It lets users define validation rules once and apply them across multiple dataset collections - supporting consistent, accurate data checks.
+
+__The DVE offers__:
+
+- SQL configuration-based validations
+- Format normalization to Parquet for a unified data representation
+- Data modelling and typecasting
+- Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Validate and enforce referential integrity checks with minimal configuration
+- Deriving new fields and entities
+- Clear validation reporting, including summary insights and record-level error messages
+
+As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` (data ingest schema) file and example files can be accessed [here ↗️](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/).
+
+## Installation and usage
+
+Please see the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/user_guidance/install/).
+
+## Requesting new features and raising bug reports
+**Before creating new issues, please check to see if the same bug/feature has been created already. Where a duplicate is created, the ticket will be closed and referenced to an existing issue.**
+
+If you have spotted a bug with the DVE then please raise an issue [here ↗️](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template". 
+
+If you have feature request then please follow the same process whilst using the "Feature request template".
+
+## Upcoming features
+Below is a list of features that we would like to implement or have been requested.
+| Feature                                                                         | Release Version   | Released? |
+| ------------------------------------------------------------------------------- | ----------------- | --------- |
+| Open source release                                                             | 0.1.0             | Yes       |
+| Uplift to Python 3.11                                                           | 0.2.0             | Yes       |
+| Uplift Pyspark to 3.5                                                           | 0.8.0             | Yes       |
+| Allow DVE to run on Python 3.12+                                                | 0.8.0             | Yes       |
+| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes       |
+| Upgrade DuckDB to v1.4                                                          | 0.10.0            | Yes       |
+| Uplift Pyspark to 4.0+                                                          | TBA               | No        |
+| Polars upgrade to v1+ | TBA | No |
+| DuckDB upgrade to v1.5+ | TBA | No |
+| Python 3.13 & 3.14 upgrade | TBA | No |
+| Create a more user friendly interface for building and modifying dischema files | TBA | No        |
+
+If you are interested in getting any of the unreleased features listed above available, then please read the [Contributing](#Contributing) section and then submit us a pull request.
+
+## Contributing
+Please see guidance [here ↗️](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md).
+
+## Legal
+This codebase is released under the MIT License. This covers both the codebase and any sample code in the documentation.
+
+Any HTML or Markdown documentation is [© Crown copyright](https://www.nationalarchives.gov.uk/information-management/re-using-public-sector-information/uk-government-licensing-framework/crown-copyright/) and available under the terms of the [Open Government 3.0 licence](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/).
+

data_validation_engine-0.10.0/README.md

@@ -0,0 +1,61 @@
+<h1 style="display: flex; align-items: center; gap: 10px;">
+    <img src="https://raw.githubusercontent.com/NHSDigital/data-validation-engine/616b55890306db4546177f7effac48ca241857ec/overrides/.icons/nhseng.svg" alt="" width="5%" height="100%" align="left">
+    Data Validation Engine
+</h1>
+
+![License](https://img.shields.io/github/license/NHSDigital/data-validation-engine)
+![PyPi](https://img.shields.io/pypi/v/data-validation-engine)
+![Conda](https://anaconda.org/nhs/data-validation-engine/badges/version.svg)
+[![CI Unit Tests](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_testing.yml)
+[![CI Formatting & Linting](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml/badge.svg)](https://github.com/NHSDigital/data-validation-engine/actions/workflows/ci_linting.yml)
+
+The Data Validation Engine (DVE) is a configuration-driven data validation library created and used by NHS England. It lets users define validation rules once and apply them across multiple dataset collections - supporting consistent, accurate data checks.
+
+__The DVE offers__:
+
+- SQL configuration-based validations
+- Format normalization to Parquet for a unified data representation
+- Data modelling and typecasting
+- Business-rule validations executed on supported backends such as Spark and DuckDB, with the option to add custom backends
+- Validate and enforce referential integrity checks with minimal configuration
+- Deriving new fields and entities
+- Clear validation reporting, including summary insights and record-level error messages
+
+As mentioned above, the DVE is "configuration driven" which means the majority of development for you as a user will be building a JSON document to describe how the data will be validated. The JSON document is known as a `dischema` (data ingest schema) file and example files can be accessed [here ↗️](https://github.com/NHSDigital/data-validation-engine/tree/main/tests/testdata). If you'd like to learn more about JSON document and how to build one from scratch, then please read the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/).
+
+## Installation and usage
+
+Please see the documentation [here ↗️](https://nhsdigital.github.io/data-validation-engine/user_guidance/install/).
+
+## Requesting new features and raising bug reports
+**Before creating new issues, please check to see if the same bug/feature has been created already. Where a duplicate is created, the ticket will be closed and referenced to an existing issue.**
+
+If you have spotted a bug with the DVE then please raise an issue [here ↗️](https://github.com/nhsengland/Data-Validation-Engine/issues) using the "bug template". 
+
+If you have feature request then please follow the same process whilst using the "Feature request template".
+
+## Upcoming features
+Below is a list of features that we would like to implement or have been requested.
+| Feature                                                                         | Release Version   | Released? |
+| ------------------------------------------------------------------------------- | ----------------- | --------- |
+| Open source release                                                             | 0.1.0             | Yes       |
+| Uplift to Python 3.11                                                           | 0.2.0             | Yes       |
+| Uplift Pyspark to 3.5                                                           | 0.8.0             | Yes       |
+| Allow DVE to run on Python 3.12+                                                | 0.8.0             | Yes       |
+| Upgrade to Pydantic 2.0                                                         | 0.9.0             | Yes       |
+| Upgrade DuckDB to v1.4                                                          | 0.10.0            | Yes       |
+| Uplift Pyspark to 4.0+                                                          | TBA               | No        |
+| Polars upgrade to v1+ | TBA | No |
+| DuckDB upgrade to v1.5+ | TBA | No |
+| Python 3.13 & 3.14 upgrade | TBA | No |
+| Create a more user friendly interface for building and modifying dischema files | TBA | No        |
+
+If you are interested in getting any of the unreleased features listed above available, then please read the [Contributing](#Contributing) section and then submit us a pull request.
+
+## Contributing
+Please see guidance [here ↗️](https://github.com/NHSDigital/data-validation-engine/blob/main/CONTRIBUTE.md).
+
+## Legal
+This codebase is released under the MIT License. This covers both the codebase and any sample code in the documentation.
+
+Any HTML or Markdown documentation is [© Crown copyright](https://www.nationalarchives.gov.uk/information-management/re-using-public-sector-information/uk-government-licensing-framework/crown-copyright/) and available under the terms of the [Open Government 3.0 licence](https://www.nationalarchives.gov.uk/doc/open-government-licence/version/3/).

{data_validation_engine-0.8.5 → data_validation_engine-0.10.0}/pyproject.toml

@@ -7,11 +7,11 @@ authors = [
 ]
 readme = "README.md"
 classifiers = [
+    "Development Status :: 4 - Beta",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
-    "Operating System :: OS Independent",
+    "Operating System :: POSIX :: Linux",
     "Topic :: Software Development :: Libraries",
     "Typing :: Typed",
 ]
@@ -25,17 +25,17 @@ Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
 Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
 
 [tool.poetry]
-version = "0.8.5"
+version = "0.10.0"
 packages = [
     { include = "dve", from = "src" },
 ]
 
 [tool.poetry.dependencies]
-python = ">=3.10,<3.13"  # breaking changes beyond 3.12
+python = ">=3.11,<3.13"  # breaking changes beyond 3.12
 boto3 = ">=1.34.162,<1.36"  # breaking change beyond 1.36
 botocore = ">=1.34.162,<1.36"  # breaking change beyond 1.36
 delta-spark = ">=3.0.0,<=3.2.0"
-duckdb = "1.1.3"  # breaking changes beyond 1.1
+duckdb = ">=1.4,<1.4.5"
 Jinja2 = "3.1.6"
 lxml = "6.1.1"
 numpy = "1.26.4"
@@ -43,10 +43,10 @@ openpyxl = "3.1.5"
 pandas = "2.3.3"
 polars = "0.20.31"
 pyarrow = "23.0.1"
-pydantic = "1.10.19"
+pydantic = "2.13.4"
 pyspark = ">=3.5.0,<=3.5.5"
-typing_extensions = "4.15.0"
-urllib3 = "2.7.0"  # dependency of boto3 & botocore
+typing_extensions = "4.16.0"
+urllib3 = "2.8.0"  # dependency of boto3 & botocore
 
 [tool.poetry.group.dev]
 optional = true
@@ -59,8 +59,11 @@ include-groups = [
 commitizen = "4.9.1"
 pre-commit = "4.3.0"
 charset-normalizer = "3.4.6"
-python-discovery = "1.2.0"
+python-discovery = "1.6.1"
 idna = "3.15"  # dependency of requests
+virtualenv = "21.14.9"
+certifi = "2026.10.23"
+filelock = "4.2.1"
 
 [tool.poetry.group.test]
 optional = true
@@ -71,7 +74,7 @@ coverage = "7.11.0"
 moto = {extras = ["s3"], version = "4.2.14"}
 cryptography = "50.0.0"  # dependency of `moto`
 requests = "2.33.0"  # dependency of `moto`
-Werkzeug = "3.1.6"
+Werkzeug = "3.1.9"
 pytest = "9.0.3"
 pytest-lazy-fixtures = "1.4.0"  # switched from https://github.com/TvoroG/pytest-lazy-fixture as it's no longer supported
 xlsx2csv = "0.8.4"  # polars requirement
@@ -80,11 +83,12 @@ xlsx2csv = "0.8.4"  # polars requirement
 optional = true
 
 [tool.poetry.group.lint.dependencies]
-black = "26.3.1"
+black = "26.10.1"
 astroid = "3.3.9"
 isort = "5.13.2"
 pylint = "3.3.9"
-mypy = "1.11.2"
+mypy = "1.20.2"
+librt = "0.11.0"  # mypy dependency
 boto3-stubs = {extras = ["essential"], version = "1.26.72"}
 botocore-stubs = "1.29.72"
 pandas-stubs = "1.2.0.62"
@@ -107,7 +111,7 @@ mkdocs = "1.6.1"
 mkdocstrings = { version = "1.0.3", extras = ["python"] }
 griffelib = "2.0.1"
 pymdown-extensions = "11.0.1"
-zensical = "0.0.46"
+zensical = "0.0.63"
 
 [tool.ruff]
 line-length = 100
@@ -291,7 +295,7 @@ redefining-builtins-modules = [
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-tag_format = "v$major.$minor.$patch"
+tag_format = "v$major.$minor.${patch}$prerelease"
 version_scheme = "pep440"
 version_provider = "poetry"
 update_changelog_on_bump = true

{data_validation_engine-0.8.5 → data_validation_engine-0.10.0}/src/dve/common/error_utils.py

@@ -5,7 +5,7 @@ import json
 import logging
 from collections.abc import Iterable
 from itertools import chain
-from multiprocessing import Queue
+from queue import Queue
 from threading import Thread
 from typing import Optional, Union
 

{data_validation_engine-0.8.5 → data_validation_engine-0.10.0}/src/dve/core_engine/backends/base/auditing.py

@@ -15,7 +15,7 @@ from queue import Queue as ThreadQueue
 from types import TracebackType
 from typing import Any, ClassVar, Generic, Optional, TypeVar, Union
 
-from pydantic import ValidationError, validate_arguments
+from pydantic import ValidationError, validate_call
 from typing_extensions import Literal, get_origin
 
 from dve.core_engine.models import (
@@ -99,8 +99,8 @@ class BaseAuditor(Generic[AuditReturnType]):
     def schema(self) -> dict[str, type]:
         """Determine python schema of auditor"""
         return {
-            fld: str if get_origin(mdl.type_) == Literal else mdl.type_
-            for fld, mdl in self._record_type.__fields__.items()
+            fld: str if get_origin(mdl.annotation) == Literal else mdl.annotation  # type: ignore
+            for fld, mdl in self._record_type.model_fields.items()
         }
 
     @staticmethod
@@ -196,7 +196,7 @@ class BaseAuditingManager(
         """Convert AuditReturnType to iterable of dictionaries"""
         raise NotImplementedError()
 
-    @validate_arguments
+    @validate_call
     def add_processing_records(self, processing_records: list[ProcessingStatusRecord]):
         """Add an entry to the processing_status auditor."""
         if self.pool:
@@ -208,7 +208,7 @@ class BaseAuditingManager(
             records=[dict(rec) for rec in processing_records]
         )
 
-    @validate_arguments
+    @validate_call
     def add_submission_statistics_records(self, sub_stats: list[SubmissionStatisticsRecord]):
         """Add an entry to the submission statistics auditor."""
         if self.pool:
@@ -218,7 +218,7 @@ class BaseAuditingManager(
             )
         return self._submission_statistics.add_records(records=[dict(rec) for rec in sub_stats])
 
-    @validate_arguments
+    @validate_call
     def add_transfer_records(self, transfer_records: list[TransferRecord]):
         """Add an entry to the transfers auditor"""
         if self.pool:
@@ -227,7 +227,7 @@ class BaseAuditingManager(
             )
         return self._transfers.add_records(records=[dict(rec) for rec in transfer_records])
 
-    @validate_arguments
+    @validate_call
     def add_new_submissions(
         self,
         submissions: list[SubmissionMetadata],
@@ -250,7 +250,7 @@ class BaseAuditingManager(
                     processing_status="received",
                     job_run_id=job_run_id,
                     **ts_info,
-                ).dict(),
+                ).model_dump(),
             }
             processing_status_recs.append(processing_rec)
             if sub_info:

{data_validation_engine-0.8.5 → data_validation_engine-0.10.0}/src/dve/core_engine/backends/base/contract.py

@@ -443,10 +443,6 @@ class BaseDataContract(Generic[EntityType], ABC):
                 ],
             )
 
-        if contract_metadata.cache_originals:
-            for entity_name in list(entities):
-                entities[f"Original{entity_name}"] = entities[entity_name]
-
         return entities, feedback_errors_uri, successful, processing_errors_uri
 
     def read_parquet(self, path: URI, **kwargs) -> EntityType:

{data_validation_engine-0.8.5 → data_validation_engine-0.10.0}/src/dve/core_engine/backends/base/reader.py

@@ -8,8 +8,16 @@ from typing import Any, ClassVar, Optional, TypeVar
 from pydantic import BaseModel
 from typing_extensions import Protocol
 
-from dve.core_engine.backends.exceptions import MessageBearingError, ReaderLacksEntityTypeSupport
+from dve.core_engine.backends.exceptions import (
+    CriticalMessageBearingError,
+    MessageBearingError,
+    ReaderLacksEntityTypeSupport,
+)
 from dve.core_engine.backends.types import EntityName, EntityType
+from dve.core_engine.configuration.v1 import (
+    AllowedAdditionalReaderChecks,
+    _ReaderAdditionalChecksConfig,
+)
 from dve.core_engine.message import FeedbackMessage
 from dve.core_engine.type_hints import URI, ArbitraryFunction, WrapDecorator
 from dve.parser.file_handling.service import open_stream
@@ -65,6 +73,10 @@ class BaseFileReader(ABC):
     decorated with the '@read_function' decorator, and is used in `read_entity_type`.
 
     """
+    ft_error_code: Optional[str] = "MalformedFile"
+    """Default error code for when a file/submission cannot be parsed succesfully."""
+    ft_error_message: Optional[str] = "The resource doesn't seem to be a valid text file"
+    """Default error message for when a file/submission cannot be parsed succesfully."""
 
     def __init_subclass__(cls, *_, **__) -> None:
         """When this class is subclassed, create and populate the `__read_methods__`
@@ -109,7 +121,10 @@ class BaseFileReader(ABC):
         entity_name: EntityName,
         schema: type[BaseModel],
         all_model_fields: Optional[set[str]] = None,
-    ) -> EntityType:
+        additional_checks: Optional[
+            dict[AllowedAdditionalReaderChecks, _ReaderAdditionalChecksConfig]
+        ] = None,
+    ):
         """Read to the specified entity type, if supported.
 
         NOTE: Simple types should either be returned as strings (if present) or
@@ -117,21 +132,47 @@ class BaseFileReader(ABC):
         data contract.
 
         """
-        if entity_name == Iterator[dict[str, Any]]:
-            return self.read_to_py_iterator(
+        additional_checks = additional_checks or {}
+
+        self.raise_if_not_sensible_file(resource, entity_name)
+
+        if entity_type == Iterator[dict[str, Any]]:
+            entity = self.read_to_py_iterator(
                 resource, entity_name, schema, all_model_fields  # type: ignore
             )
 
-        self.raise_if_not_sensible_file(resource, entity_name)
+        else:
 
-        try:
-            reader_func = self.__read_methods__[entity_type]
-        except KeyError as err:
-            raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
+            try:
+                reader_func = self.__read_methods__[entity_type]
+            except KeyError as err:
+                raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
 
-        return reader_func(
-            self, resource, entity_name, schema, all_model_fields=all_model_fields  # type: ignore
-        )
+            entity = reader_func(
+                self,
+                resource,
+                entity_name,
+                schema,
+                all_model_fields=all_model_fields,  # type: ignore
+            )
+
+        if config := additional_checks.get("check_empty"):
+            if self.check_entity_empty(entity):
+                raise MessageBearingError(
+                    f"The mandatory entity {entity_name} is empty",
+                    messages=[
+                        FeedbackMessage(
+                            entity=entity_name,
+                            record=None,
+                            failure_type="submission",
+                            error_location=entity_name,
+                            error_code=config.error_code,
+                            error_message=config.error_message,
+                        )
+                    ],
+                )
+
+        return entity
 
     def add_record_index(self, entity: EntityType, **kwargs) -> EntityType:
         """Add a record index to the entity"""
@@ -141,6 +182,10 @@ class BaseFileReader(ABC):
         """Drop a record index to the entity"""
         raise NotImplementedError(f"drop_record_index not implemented in {self.__class__}")
 
+    def check_entity_empty(self, entity: EntityType) -> bool:
+        """Determine if the entity supplied is empty"""
+        raise NotImplementedError(f"check_entity_empty not implemented in {self.__class__}")
+
     def write_parquet(
         self,
         entity: EntityType,
@@ -171,20 +216,22 @@ class BaseFileReader(ABC):
             return False
         return True
 
-    def raise_if_not_sensible_file(self, resource: URI, entity_name: str):
+    def raise_if_not_sensible_file(
+        self,
+        resource: URI,
+        entity_name: str,
+    ):
         """Sense check that the file is a text file. Raise error if doesn't
         appear to be the case."""
         if not self._check_likely_text_file(resource):
-            raise MessageBearingError(
+            raise CriticalMessageBearingError(
                 "The submitted file doesn't appear to be text",
-                messages=[
-                    FeedbackMessage(
-                        entity=entity_name,
-                        record=None,
-                        failure_type="submission",
-                        error_location="Whole File",
-                        error_code="MalformedFile",
-                        error_message="The resource doesn't seem to be a valid text file",
-                    )
-                ],
+                message=FeedbackMessage(
+                    entity=entity_name,
+                    record=None,
+                    failure_type="submission",
+                    error_location="Whole File",
+                    error_code=self.ft_error_code,
+                    error_message=self.ft_error_message,
+                ),
             )