data-validation-engine 0.7.0__tar.gz → 0.7.2__tar.gz

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/PKG-INFO

@@ -1,15 +1,16 @@
 Metadata-Version: 2.4
 Name: data-validation-engine
-Version: 0.7.0
+Version: 0.7.2
 Summary: `nhs data validation engine` is a framework used to validate data
+License-Expression: MIT
 License-File: LICENSE
 Author: NHS England
 Author-email: england.contactus@nhs.net
 Requires-Python: >=3.10,<3.12
-Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
+Classifier: Operating System :: OS Independent
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Typing :: Typed
 Requires-Dist: Jinja2 (==3.1.*)
@@ -22,9 +23,14 @@ Requires-Dist: openpyxl (>=3.1,<4.0)
 Requires-Dist: pandas (>=2.2.2,<3.0.0)
 Requires-Dist: polars (==0.20.*)
 Requires-Dist: pyarrow (>=17.0.0,<18.0.0)
-Requires-Dist: pydantic (==1.10.15)
+Requires-Dist: pydantic (==1.10.16)
 Requires-Dist: pyspark (==3.4.*)
 Requires-Dist: typing_extensions (>=4.6.2,<5.0.0)
+Project-URL: Changelog, https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md
+Project-URL: Documentation, https://nhsdigital.github.io/data-validation-engine/
+Project-URL: Homepage, https://github.com/NHSDigital/data-validation-engine
+Project-URL: Issues, https://github.com/NHSDigital/data-validation-engine/issues
+Project-URL: Repository, https://github.com/NHSDigital/data-validation-engine.git
 Description-Content-Type: text/markdown
 
 <h1 style="display: flex; align-items: center; gap: 10px;">

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/pyproject.toml

@@ -1,12 +1,11 @@
-[tool.poetry]
+[project]
 name = "data-validation-engine"
-version = "0.7.0"
+dynamic = [ "version" ]
 description = "`nhs data validation engine` is a framework used to validate data"
-authors = ["NHS England <england.contactus@nhs.net>"]
-readme = "README.md"
-packages = [
-    { include = "dve", from = "src" },
+authors = [
+    { name = "NHS England", email = "england.contactus@nhs.net" }
 ]
+readme = "README.md"
 classifiers = [
     "Programming Language :: Python :: 3",
     "Programming Language :: Python :: 3.10",
@@ -15,6 +14,20 @@ classifiers = [
     "Topic :: Software Development :: Libraries",
     "Typing :: Typed",
 ]
+license = "MIT"
+
+[project.urls]
+Homepage = "https://github.com/NHSDigital/data-validation-engine"
+Documentation = "https://nhsdigital.github.io/data-validation-engine/"
+Repository = "https://github.com/NHSDigital/data-validation-engine.git"
+Issues = "https://github.com/NHSDigital/data-validation-engine/issues"
+Changelog = "https://github.com/NHSDigital/data-validation-engine/blob/main/CHANGELOG.md"
+
+[tool.poetry]
+version = "0.7.2"
+packages = [
+    { include = "dve", from = "src" },
+]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.12"
@@ -28,7 +41,7 @@ openpyxl = "^3.1"
 pandas = "^2.2.2"
 polars = "0.20.*"
 pyarrow = "^17.0.0"
-pydantic = "1.10.15"
+pydantic = "1.10.16"
 pyspark = "3.4.*"
 typing_extensions = "^4.6.2"
 
@@ -78,6 +91,15 @@ types-setuptools = "68.2.0.0"
 types-urllib3 = "1.26.25.14"
 types-xmltodict = "0.13.0.3"
 
+[tool.poetry.group.docs]
+optional = true
+
+[tool.poetry.group.docs.dependencies]
+click = "8.2.1"
+mkdocs = "^1.6.1"
+mkdocstrings = { version = "^1.0.3", extras = ["python"] }
+zensical = "~=0.0.23"
+
 [tool.ruff]
 line-length = 100
 

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/src/dve/core_engine/backends/implementations/duckdb/readers/csv.py

@@ -82,11 +82,12 @@ class DuckDBCSVReader(BaseFileReader):
                 messages=[
                     FeedbackMessage(
                         entity=entity_name,
-                        record=None,
+                        record={"missing_fields": missing},
                         failure_type="submission",
                         error_location="Whole File",
+                        reporting_field="missing_fields",
                         error_code=self.field_check_error_code,
-                        error_message=f"{self.field_check_error_message} - missing fields: {missing}",  # pylint: disable=line-too-long
+                        error_message=f"{self.field_check_error_message}",  # pylint: disable=line-too-long
                     )
                 ],
             )
@@ -202,6 +203,7 @@ class DuckDBCSVRepeatingHeaderReader(PolarsToDuckDBCSVReader):
     `NonDistinctHeaderError`.
 
     So using the example above, the expected entity would look like this...
+
     | headerCol1 | headerCol2 | headerCol3 |
     | ---------- | ---------- | ---------- |
     | shop1      | clothes    | 2025-01-01 |

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/src/dve/core_engine/backends/implementations/duckdb/rules.py

@@ -505,7 +505,7 @@ class DuckDBStepImplementations(BaseStepImplementations[DuckDBPyRelation]):
             messages.append(
                 FeedbackMessage(
                     entity=config.reporting.reporting_entity_override or config.entity_name,
-                    original_entity=config.entity_name,
+                    original_entity=config.reporting.original_entity_override or config.entity_name,
                     record=record,  # type: ignore
                     error_location=config.reporting.legacy_location,
                     error_message=template_object(config.reporting.message, record),  # type: ignore

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/src/dve/core_engine/backends/implementations/spark/rules.py

@@ -401,7 +401,7 @@ class SparkStepImplementations(BaseStepImplementations[DataFrame]):
                 # more complex extraction done in reporting module
                 FeedbackMessage(
                     entity=config.reporting.reporting_entity_override or config.entity_name,
-                    original_entity=config.entity_name,
+                    original_entity=config.reporting.original_entity_override or config.entity_name,
                     record=record.asDict(recursive=True),
                     error_location=config.reporting.legacy_location,
                     error_message=template_object(

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/src/dve/core_engine/backends/metadata/reporting.py

@@ -10,6 +10,7 @@ from typing_extensions import Literal
 
 from dve.core_engine.templating import template_object
 from dve.core_engine.type_hints import (
+    EntityName,
     ErrorCategory,
     ErrorCode,
     ErrorEmitValue,
@@ -102,6 +103,12 @@ class BaseReportingConfig(BaseModel):
     If this is not provided, the top level names from the `location` field
     will be used in the report.
 
+    """
+    original_entity_override: Optional[EntityName] = None
+    """
+    This should only be populated where you need to refer back to an originally defined entity that
+    contains the key field(s) of interest. This might be required in cases where you've created
+    lots of new entities during a complex rule.
     """
 
     def template(

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/src/dve/core_engine/configuration/v1/filters.py

@@ -24,6 +24,7 @@ class ConcreteFilterConfig(BaseModel):
     reporting_field: Optional[Union[str, list[str]]] = None
     reporting_field_name: Optional[str] = None
     category: ErrorCategory = "Bad value"
+    original_entity_override: Optional[str] = None
 
     def to_step(self) -> AbstractStep:
         """Create a deferred filter from the concrete filter config."""
@@ -37,6 +38,7 @@ class ConcreteFilterConfig(BaseModel):
             legacy_is_informational=self.is_informational,
             legacy_location=self.error_location,
             legacy_reporting_field=self.reporting_field,
+            original_entity_override=self.original_entity_override,
         )
 
         return DeferredFilter(

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/src/dve/pipeline/foundry_ddb_pipeline.py

@@ -42,11 +42,15 @@ class FoundryDDBPipeline(DDBDVEPipeline):
             write_to.parent.mkdir(parents=True, exist_ok=True)
             write_to = write_to.as_posix()
         self.write_parquet(  # type: ignore # pylint: disable=E1101
-            self._audit_tables._processing_status.get_relation(),  # pylint: disable=W0212
+            self._audit_tables._processing_status.get_relation().filter( # pylint: disable=W0212
+                f"submission_id = '{submission_info.submission_id}'"
+            ),
             fh.joinuri(write_to, "processing_status.parquet"),
         )
         self.write_parquet(  # type: ignore # pylint: disable=E1101
-            self._audit_tables._submission_statistics.get_relation(),  # pylint: disable=W0212
+            self._audit_tables._submission_statistics.get_relation().filter( # pylint: disable=W0212
+                f"submission_id = '{submission_info.submission_id}'"
+            ),
             fh.joinuri(write_to, "submission_statistics.parquet"),
         )
         return write_to

{data_validation_engine-0.7.0 → data_validation_engine-0.7.2}/src/dve/reporting/excel_report.py

@@ -135,8 +135,11 @@ class SummaryItems:
         if "Status" not in self.summary_dict:
             summary.append(["", "Status", status])
 
+        _key_renames = {
+            "File Size": "File Size (Bytes)",
+        }
         for key, value in self.summary_dict.items():
-            summary.append(["", key, str(value)])
+            summary.append(["", _key_renames.get(key, key), str(value)])
 
         summary.append(["", ""])