data-validation-engine 0.6.2__py3-none-any.whl

dve/metadata_parser/exc.py

@@ -0,0 +1,44 @@
+"""Exceptions around entities and for use in validators"""
+
+
+class EntityNotFoundError(KeyError):
+    """Error for missing entities"""
+
+
+class LocWarning(UserWarning):
+    """Warning class with optional location parameter"""
+
+    def __init__(self, msg, loc=""):
+        self.msg = msg
+        self.loc = loc
+        super().__init__()
+
+    def __str__(self):
+        """string representation of a loc warning"""
+        return f"{self.loc} {self.msg}"
+
+
+class RecordRejection(ValueError):
+    """Exception to indicate a record has been rejected"""
+
+
+class FileRejection(ValueError):
+    """Exception to indicate a file has been rejected"""
+
+
+class GroupRejection(ValueError):
+    """Exception to indicate a chain of records have been rejected
+    normally used in a referential integrity check or for duplicated records
+    """
+
+
+class TypeNotFoundError(KeyError):
+    """Exception raised when unable to locate correct type from passed in metadata"""
+
+
+ERRORS = {
+    "recordrejection": RecordRejection,
+    "filerejection": FileRejection,
+    "warning": LocWarning,
+    "grouprejection": GroupRejection,
+}

dve/metadata_parser/function_library.py

@@ -0,0 +1,64 @@
+"""Function library for wrapping into pydantic validators"""
+
+import functools
+
+import pydantic
+
+
+def _nullcheck(func):
+    """Decorator to nullcheck incoming values from pydantic validators
+    this removes the need to add a nullcheck to the value for each function.
+    Nullchecks may still be needed for any additional values passed in.
+
+    e.g.
+    ```python
+    @_nullcheck # nullchecks value but not other_field
+    def func(value, other_field):
+        if other_field is None:
+            # do null thing
+        # logic
+    ```
+    """
+
+    @functools.wraps(func)
+    def inner(value, *args, **kwargs):
+        if value is None or not str(value).strip():
+            return None
+        return func(value, *args, **kwargs)
+
+    return inner
+
+
+# demo function
+@_nullcheck
+@pydantic.validate_arguments
+def normalise(value, capitalize: bool = False):  # pragma: no cover
+    """Normalises a string by capitalising it"""
+    if capitalize:
+        return str(value).capitalize()
+    return value
+
+
+@_nullcheck
+def exclude_word(value, word: str):
+    """Returns None if the word is present in the value"""
+    if word.lower() in str(value).lower():
+        return None
+    return value
+
+
+@_nullcheck
+@pydantic.validate_arguments
+def split(value, split_on: str, keep: int = 0):
+    """Splits a string on a given delimiter and keeps only the value at the given index
+    defaults to 0
+    """
+    try:
+        return str(value).split(split_on)[keep]
+    except IndexError as exc:
+        raise ValueError from exc
+
+
+def static_key(value):  # pylint: disable=W0613
+    """Return a fixed value, for use as a static join key"""
+    return 1

dve/metadata_parser/function_wrapper.py

@@ -0,0 +1,201 @@
+"""Wrapping functions for wrapping generic functions"""
+
+import warnings
+from collections.abc import Callable, Iterable
+from typing import Any, Optional, Union
+
+import pydantic
+
+from dve.metadata_parser import exc
+
+PydanticCompatible = Callable[
+    [Any, dict[str, Any], pydantic.fields.ModelField, pydantic.BaseConfig], Any
+]
+"""Function Compatable with pydantic
+        Args:
+            value (Any): Value to be validated
+            values (dict[str, Any]): dict of previously validated fields
+            field (pydantic.fields.ModelField): field object containing field name and type
+            config (pydantic.BaseConfig): the config that determines things like aliases
+
+"""
+
+
+def error_handler(
+    error_type: Union[type[Exception], type[Warning]],
+    error_message: str,
+    field: pydantic.fields.ModelField,
+):
+    """Determines whether to raise an error or warning based on error_type
+
+    Args:
+        error_type (Union[type[Exception], type[Warning]]): type of error to raise
+        error_message (str): message to apply
+        field (pydantic.fields.ModelField): field that caused the error to be raised
+
+    Raises:
+        error_type
+
+    """
+    if issubclass(error_type, exc.LocWarning):
+        warnings.warn(error_type(msg=error_message, loc=field.name))
+    elif issubclass(error_type, Warning):
+        warnings.warn(error_message, error_type)
+    else:
+        raise error_type(error_message)
+
+
+def pydantic_wrapper(
+    error_type: Union[type[Exception], type[Warning]],
+    error_message: str,
+    *field_names: str,
+    failure_function: Callable = lambda x: x is False,
+    return_result: bool = True,
+    **kwargs,
+) -> Callable[[Callable], PydanticCompatible]:
+    """Wraps generic functions and returns a pydantic compatible function
+    takes an error type and error_message to allow the failure of a function to be customised
+
+    takes field_names of fields to be included into the generic function. should be in order
+    that they appear in the function.
+
+    takes a function that will result in the passed exception being raised
+
+    Args:
+        error_type (type[Exception]): The exception type to be raised if the failure_function
+        evaluates to True
+        error_message (str): Message to be passed to the above exception
+        failure_function (Optional[Callable]): A callable that when it evaluates to True
+        raises the above exception. Defaults to lambda x: x is False.
+        return_result (bool): Whether to return the result from the wrapped function or the
+        or the original value that was passed.
+        True should be used when the function transforms the data in some way
+        e.g. stripping whitespace from NHS numbers.
+        False should be used when the function is a comparison
+        e.g. x > y. in this case the value of the field is returned rather than the bool.
+
+    """
+
+    def wrapper(
+        func: Callable,
+    ) -> PydanticCompatible:
+        """Wraps the passed function and returns a function with a pydantic compatible calling
+        signature
+
+        Args:
+            func (Callable): function to be wrapped
+
+        Raises:
+            error_type: error passed in with the message given
+
+        Returns:
+            Callable: wrapped function with call signature:
+            (value: Any, values: dict, field: ModelField, config: BaseConfig) -> Any
+
+        """
+
+        def inner(
+            value: Any,
+            values: dict[str, Any],
+            field: pydantic.fields.ModelField,  # pylint: disable=unused-argument
+            config: pydantic.BaseConfig,  # pylint: disable=unused-argument
+        ) -> Any:
+            fields = [values.get(name) for name in field_names]
+            result = None
+            try:
+                result = func(value, *fields, **kwargs)
+            except (ValueError, TypeError, AssertionError):
+                error_handler(error_type, error_message, field)
+            else:
+                # only want to check if the return is False if we are returning the value
+                if not return_result and failure_function(result):
+                    error_handler(error_type, error_message, field)
+            if return_result:
+                return result
+            return value
+
+        inner.__name__ = f"wrapped_{func.__name__}"
+        return inner
+
+    return wrapper
+
+
+validator_args = pydantic.validator.__kwdefaults__.copy()
+
+
+def create_validator(
+    function: Callable,
+    field: str,
+    error_type: type[Exception],
+    error_message: str,
+    fields: Optional[Iterable[str]] = None,
+    return_result=True,
+    **kwargs,
+):
+    """Creates a pydantic_validator from a function
+
+    Args:
+        function (Callable): function to wrap
+        field (str): field validator is applier to
+        fields (Iterable[str]): other fields to be included in validation (in order of arguments)
+        error_type (type[Exception]): Error to be raised on failure
+        error_message (str): Message to be raised on failure
+        kwargs:
+            pydantic_wrapper_kwargs:
+                failure_function: function called to cause the function to raise the passed
+                error when evaluates to True
+                return_result: bool = True: whether to return the result of the wrapped function
+                or the unchanged value
+
+            pydantic.validator kwargs:
+                pre: bool = False
+                each_item: bool = False
+                always: bool = False
+                check_fields: bool = True
+                whole: bool = None
+                allow_reuse: bool = False
+
+            function kwargs
+
+    Returns:
+        classmethod: wrapped function wrapped by pydantics validator
+
+    """
+    validator_kwargs = _validator_kwargs(
+        **{key: kwargs.pop(key, default) for key, default in validator_args.items()}
+    )
+
+    if fields is None:
+        fields = []
+
+    wrapped = pydantic_wrapper(
+        error_type,
+        error_message,
+        *fields,
+        return_result=return_result,
+        **kwargs,
+    )(function)
+
+    validator_kwargs.update(allow_reuse=True)
+    validator = pydantic.validator(field, **validator_kwargs)(wrapped)
+    return validator
+
+
+@pydantic.validate_arguments
+def _validator_kwargs(
+    pre: bool = False,
+    each_item: bool = False,
+    always: bool = False,
+    check_fields: bool = True,
+    whole: Optional[bool] = None,
+    allow_reuse: bool = False,
+    **kwargs,  # pylint: disable=unused-argument
+):
+    return {
+        "pre": pre,
+        "each_item": each_item,
+        "always": always,
+        "check_fields": check_fields,
+        "whole": whole,
+        "allow_reuse": allow_reuse,
+    }

dve/metadata_parser/model_generator.py

@@ -0,0 +1,119 @@
+"""Tools for parsing metadata to pydantic Models"""
+
+# pylint: disable=super-init-not-called
+import warnings
+from abc import ABCMeta, abstractmethod
+from collections.abc import Mapping
+from copy import deepcopy
+
+# This _needs_ to be `typing.Mapping`, or pydantic complains.
+from typing import Any, Optional, Union
+
+import pydantic as pyd
+from typing_extensions import Literal
+
+from dve.metadata_parser import domain_types
+from dve.metadata_parser.models import DatasetSpecification
+from dve.metadata_parser.utilities import FieldTypeOption
+
+# Type doesn't like Ellipsis. the type hint this is used in doesn't like type(Ellipsis)
+EllipsisType = Any
+"""3.9 compatible ellipsis type"""
+
+TypeAnnotation = Union[type, Any]  # pylint: disable=invalid-name
+"""An arbitrary type annotation"""
+TableType = Literal["schema", "table"]
+"""
+The type of 'table' specified in the schema.
+
+Values:
+
+ - 'table': the table is a 'real' table, and should be expected
+   to be created by the engine.
+ - 'schema': the 'table' is a schema definition for a complex field
+   within a table. These should be defined before the tables they
+   are used in.
+
+"""
+
+
+@pyd.validate_arguments
+def constr(
+    *,
+    strip_whitespace: bool = False,
+    to_lower: bool = False,
+    strict: bool = False,
+    min_length: Optional[int] = None,
+    max_length: Optional[int] = None,
+    curtail_length: Optional[int] = None,
+    regex: Optional[str] = None,
+):
+    """Wrapper around constr to enable argument validation"""
+    return pyd.constr(
+        strip_whitespace=strip_whitespace,
+        to_lower=to_lower,
+        strict=strict,
+        min_length=min_length,  # type: ignore
+        max_length=max_length,  # type: ignore
+        curtail_length=curtail_length,  # type: ignore
+        regex=regex,  # type: ignore
+    )
+
+
+STR_TO_PY_MAPPING: Mapping[str, FieldTypeOption] = {
+    "constr": constr,
+    "conint": pyd.validate_arguments(pyd.conint),
+    "condate": pyd.validate_arguments(pyd.condate),
+    "condecimal": pyd.validate_arguments(pyd.condecimal),
+    "postcode": domain_types.Postcode,
+    "nhsnumber": domain_types.NHSNumber,
+    "permissivenhsno": domain_types.permissive_nhs_number(),
+    "alphanumeric": domain_types.alphanumeric,
+    "identifier": domain_types.identifier,
+    "orgid": domain_types.OrgID,
+    "formatteddatetime": domain_types.formatteddatetime,
+    "formattedtime": domain_types.formattedtime,
+    "conformatteddate": domain_types.conformatteddate,
+    "reportingperiodstart": domain_types.reportingperiod(reporting_period_type="start"),
+    "reportingperiodend": domain_types.reportingperiod(reporting_period_type="end"),
+}
+
+
+class ModelLoader(metaclass=ABCMeta):  # pylint: disable=too-few-public-methods
+    """An abstract model loader."""
+
+    @abstractmethod
+    def __init__(self, contract_contents: dict[str, Any], type_map: Optional[dict] = None):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def generate_models(
+        self, additional_validators: Optional[dict] = None
+    ) -> dict[str, pyd.main.ModelMetaclass]:
+        """Generates models from the instance schema.
+
+        Args:
+            additional_validators (Optional[dict], optional): Any validation rules aside from
+            those described in the schema. Defaults to None [DEPRECATED]
+
+        Returns:
+            dict[str, model]: dict of table names to pydantic models
+
+        """
+        raise NotImplementedError()
+
+
+class JSONtoPyd(ModelLoader):  # pylint: disable=too-few-public-methods
+    """Generate pydantic model from a JSON schema."""
+
+    def __init__(self, contract_contents: dict[str, Any], type_map: Optional[dict] = None):
+        self.contract_contents = contract_contents
+        self.type_map = deepcopy(type_map or STR_TO_PY_MAPPING)
+
+    def generate_models(
+        self, additional_validators: Optional[dict] = None
+    ) -> dict[str, pyd.main.ModelMetaclass]:
+        """Generates pydantic models from a loaded json file"""
+        if additional_validators:
+            warnings.warn("Ignoring additional validator functions")
+        return DatasetSpecification(**self.contract_contents).load_models(self.type_map)