data-validation-engine 0.6.2__py3-none-any.whl

dve/core_engine/backends/base/core.py

@@ -0,0 +1,124 @@
+"""Core functionality for the backend bases."""
+
+from collections.abc import Callable, Iterator, Mapping, MutableMapping
+from typing import Any, Generic, Optional
+
+from typing_extensions import get_args, get_origin
+
+from dve.core_engine.backends.exceptions import ConstraintError, MissingEntity, MissingRefDataEntity
+from dve.core_engine.backends.types import EntityType
+from dve.core_engine.type_hints import EntityName
+
+get_original_bases: Callable[[type], tuple[Any, ...]]
+try:
+    # pylint: disable=ungrouped-imports
+    from typing_extensions import get_original_bases  # type: ignore
+except ImportError:
+
+    def get_original_bases(__cls: type) -> tuple[Any, ...]:
+        """A basic version of 'get_original_bases' in case it's not in typing extensions."""
+        try:
+            return __cls.__orig_bases__  # type: ignore
+        except AttributeError:
+            pass
+
+        try:
+            return __cls.__mro_entries__  # type: ignore
+        except AttributeError:
+            pass
+        return __cls.__mro__
+
+
+def get_entity_type(child: type, annotated_type_name: str) -> type[EntityType]:
+    """Get the annotated entity type from a subclass, given the name of the parent
+    class which must be annotated.
+
+    """
+    for base in get_original_bases(child):
+        if isinstance(base, type):
+            if base.__name__ != annotated_type_name:
+                continue
+        else:
+            origin = get_origin(base)
+            if origin is None or origin.__name__ != annotated_type_name:
+                continue
+
+        annotations = get_args(base)
+        if not annotations:
+            raise TypeError(f"{child}: Cannot create an untyped `{annotated_type_name}` subclass")
+        if len(annotations) != 1:
+            raise TypeError(f"{child}: `{annotated_type_name}` must have exactly one entity type")
+        return annotations[0]  # type: ignore
+
+    raise TypeError(f"{child}: No `{annotated_type_name}` parent found")
+
+
+IsRefdata = bool
+
+
+class EntityManager(Generic[EntityType], MutableMapping[EntityName, EntityType]):
+    """An entity manager that creates a copy of the entities to mutate
+    during processing and ensures that reference data is not mutated.
+
+    This also ensures appropriate errors are raised for get/sets that
+    result in nicer error logs.
+
+    """
+
+    def __init__(
+        self,
+        entities: MutableMapping[EntityName, EntityType],
+        reference_data: Optional[Mapping[EntityName, EntityType]] = None,
+    ) -> None:
+        self.entities = {}
+        """A copy of the loaded entities."""
+        for entity_name, entity in entities.items():
+            if entity_name.startswith("refdata_"):
+                raise ValueError(f"Entity name cannot start with 'refdata_', got {entity_name!r}")
+            self.entities[entity_name] = entity
+
+        self.reference_data = reference_data if reference_data is not None else {}
+        """The reference data mapping."""
+
+    @staticmethod
+    def _get_key_and_whether_refdata(key: str) -> tuple[EntityName, IsRefdata]:
+        """Get the key and whether the entity is a reference data entry."""
+        if key.startswith("refdata_"):
+            return key[8:], True
+        return key, False
+
+    def __getitem__(self, key: EntityName) -> EntityType:
+        entity_name, is_refdata = self._get_key_and_whether_refdata(key)
+
+        try:
+            if is_refdata:
+                return self.reference_data[entity_name]
+            return self.entities[entity_name]
+        except KeyError as err:
+            error_type = MissingRefDataEntity if is_refdata else MissingEntity
+            raise error_type(entity_name=entity_name) from err
+
+    def __setitem__(self, key: EntityName, value: EntityType) -> None:
+        entity_name, is_refdata = self._get_key_and_whether_refdata(key)
+        if is_refdata:
+            raise ConstraintError(
+                f"Attempting to mutate reference data entity {entity_name!r}",
+                constraint=f"reference data entry {entity_name!r} must not be mutated",
+            )
+        self.entities[entity_name] = value
+
+    def __delitem__(self, key: EntityName) -> None:
+        entity_name, is_refdata = self._get_key_and_whether_refdata(key)
+        if is_refdata:
+            raise ConstraintError(
+                f"Attempting to remove reference data entity {entity_name!r}",
+                constraint=f"reference data entry {entity_name!r} must not be mutated",
+            )
+        del self.entities[entity_name]
+
+    def __iter__(self) -> Iterator[str]:
+        yield from iter(self.entities.keys())
+        yield from ("_".join(("refdata", key)) for key in self.reference_data.keys())
+
+    def __len__(self) -> int:
+        return len(self.entities) + len(self.reference_data)

dve/core_engine/backends/base/reader.py

@@ -0,0 +1,176 @@
+"""Abstract implementation of the file parser."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Iterator
+from inspect import ismethod
+from typing import Any, ClassVar, Optional, TypeVar
+
+from pydantic import BaseModel
+from typing_extensions import Protocol
+
+from dve.core_engine.backends.exceptions import MessageBearingError, ReaderLacksEntityTypeSupport
+from dve.core_engine.backends.types import EntityName, EntityType
+from dve.core_engine.message import FeedbackMessage
+from dve.core_engine.type_hints import URI, ArbitraryFunction, WrapDecorator
+from dve.parser.file_handling.service import open_stream
+
+T = TypeVar("T")
+ET_co = TypeVar("ET_co", covariant=True)
+# This needs to be defined outside the class since otherwise mypy expects
+# BaseFileReader to be generic:
+_ReadFunctions = dict[type[T], "_UnboundReadFunction[T]"]
+"""A convenience type indicating a mapping from type to reader function."""
+_ENTITY_TYPE_ATTR_NAME = "_read_func_entity_type"
+"""The name of the read function's entity type annotation attribute."""
+
+
+class _UnboundReadFunction(Protocol[ET_co]):  # pylint: disable=too-few-public-methods
+    """The protocol required to implement a read function for a new entity type."""
+
+    @staticmethod
+    def __call__(  # pylint: disable=bad-staticmethod-argument
+        self: "BaseFileReader",  # This is the protocol for an _unbound_ method.
+        resource: URI,
+        entity_name: EntityName,
+        schema: type[BaseModel],
+    ) -> ET_co: ...
+
+
+def read_function(entity_type: T) -> WrapDecorator:
+    """A decorator function which tags read function methods within a reader class.
+    This is used to add support for different entity types in reader implementations.
+
+    """
+
+    def reader_impl_decorator(func: ArbitraryFunction) -> ArbitraryFunction:
+        """Wrap a read function to tag the entity type it implements support for."""
+        setattr(func, _ENTITY_TYPE_ATTR_NAME, entity_type)
+        return func
+
+    return reader_impl_decorator
+
+
+class BaseFileReader(ABC):
+    """An abstract representation of a reader for some file type."""
+
+    __read_methods__: ClassVar[_ReadFunctions] = {}
+    """
+    A dictionary mapping implemented entity types to their read functions.
+
+    This enables readers to implement optimised support for specific entity
+    types (rather than relying on the data contract having an optimised implementation,
+    or on the 'fallback' via a Python iterator.
+
+    This is set and populated in `__init_subclass__` by identifying methods
+    decorated with the '@read_function' decorator, and is used in `read_entity_type`.
+
+    """
+
+    def __init_subclass__(cls, *_, **__) -> None:
+        """When this class is subclassed, create and populate the `__read_methods__`
+        class variable for the subclass.
+
+        """
+        cls.__read_methods__ = {}
+
+        for attr_name in dir(cls):
+            method = getattr(cls, attr_name, None)
+            if not (ismethod(method) or callable(method)):
+                continue
+
+            entity_type: Optional[type] = getattr(method, _ENTITY_TYPE_ATTR_NAME, None)
+            if entity_type is None:
+                continue
+
+            cls.__read_methods__[entity_type] = method  # type: ignore
+
+    @abstractmethod
+    def read_to_py_iterator(
+        self,
+        resource: URI,
+        entity_name: EntityName,
+        schema: type[BaseModel],
+    ) -> Iterator[dict[str, Any]]:
+        """Iterate through the contents of the resource, yielding dicts
+        representing each record.
+
+        NOTE: Simple types should either be returned as strings (if present) or
+        `None`. Format validation, casting, and parsing should be done in the
+        data contract.
+
+        """
+        raise NotImplementedError
+
+    def read_to_entity_type(
+        self,
+        entity_type: type[EntityType],
+        resource: URI,
+        entity_name: EntityName,
+        schema: type[BaseModel],
+    ) -> EntityType:
+        """Read to the specified entity type, if supported.
+
+        NOTE: Simple types should either be returned as strings (if present) or
+        `None`. Format validation, casting, and parsing should be done in the
+        data contract.
+
+        """
+        if entity_name == Iterator[dict[str, Any]]:
+            return self.read_to_py_iterator(resource, entity_name, schema)  # type: ignore
+
+        self.raise_if_not_sensible_file(resource, entity_name)
+
+        try:
+            reader_func = self.__read_methods__[entity_type]
+        except KeyError as err:
+            raise ReaderLacksEntityTypeSupport(entity_type=entity_type) from err
+
+        return reader_func(self, resource, entity_name, schema)
+
+    def write_parquet(
+        self,
+        entity: EntityType,
+        target_location: URI,
+        schema: Optional[type[BaseModel]] = None,
+        **kwargs,
+    ) -> URI:
+        """Write entity to parquet.
+
+        NOTE: Simple types should be cast as strings (if present) or None.
+        If schema supplied then all simple types will be coerced to strings.
+
+        """
+        raise NotImplementedError(f"write_parquet not implemented in {self.__class__}")
+
+    @staticmethod
+    def _check_likely_text_file(resource: URI) -> bool:
+        """Quick sense check of file to see if it looks like text
+        - not 100% full proof, but hopefully enough to weed out most
+        non-text files"""
+        with open_stream(resource, "rb") as fle:
+            start_chunk = fle.read(4096)
+        # check for BOM character - utf-16 can contain NULL bytes
+        if start_chunk.startswith((b"\xff\xfe", b"\xfe\xff")):
+            return True
+        # if null byte in - unlikely text
+        if b"\x00" in start_chunk:
+            return False
+        return True
+
+    def raise_if_not_sensible_file(self, resource: URI, entity_name: str):
+        """Sense check that the file is a text file. Raise error if doesn't
+        appear to be the case."""
+        if not self._check_likely_text_file(resource):
+            raise MessageBearingError(
+                "The submitted file doesn't appear to be text",
+                messages=[
+                    FeedbackMessage(
+                        entity=entity_name,
+                        record=None,
+                        failure_type="submission",
+                        error_location="Whole File",
+                        error_code="MalformedFile",
+                        error_message="The resource doesn't seem to be a valid text file",
+                    )
+                ],
+            )

dve/core_engine/backends/base/reference_data.py

@@ -0,0 +1,217 @@
+"""The base implementation of the reference data loader.."""
+
+from abc import ABC, abstractmethod
+from collections.abc import Callable, Iterator, Mapping
+from typing import ClassVar, Generic, Optional, Union, get_type_hints
+
+from pydantic import BaseModel, Field
+from typing_extensions import Annotated, Literal
+
+import dve.parser.file_handling as fh
+from dve.core_engine.backends.base.core import get_entity_type
+from dve.core_engine.backends.exceptions import (
+    MissingRefDataEntity,
+    RefdataLacksFileExtensionSupport,
+)
+from dve.core_engine.backends.types import EntityType
+from dve.core_engine.type_hints import URI, EntityName
+from dve.parser.file_handling.implementations.file import LocalFilesystemImplementation
+from dve.parser.file_handling.service import _get_implementation
+
+_FILE_EXTENSION_NAME: str = "_REFDATA_FILE_EXTENSION"
+"""Name of attribute added to methods where they relate
+   to loading a particular reference file type."""
+
+
+def mark_refdata_file_extension(file_extension):
+    """Mark a method for loading a particular file extension"""
+
+    def wrapper(func: Callable):
+        setattr(func, _FILE_EXTENSION_NAME, file_extension)
+        return func
+
+    return wrapper
+
+
+class ReferenceTable(BaseModel, frozen=True):
+    """Configuration for a reference data object when table_name."""
+
+    type: Literal["table"]
+    """The object type."""
+    table_name: str
+    """Name of the table where the data persists."""
+    database: Optional[str] = None
+    """Name of the database where the reference data is located."""
+
+    @property
+    def fq_table_name(self):
+        """The fully qualified table name"""
+        if self.database:
+            return f"{self.database}.{self.table_name}"
+        return self.table_name
+
+
+class ReferenceFile(BaseModel, frozen=True):
+    """Configuration for a reference data object when a file."""
+
+    type: Literal["filename"]
+    """The object type."""
+    filename: str
+    """The path to the reference data relative to the contract."""
+
+    @property
+    def file_extension(self) -> str:
+        """The file extension of the reference file"""
+        return fh.get_file_suffix(self.filename)  # type: ignore
+
+
+class ReferenceURI(BaseModel, frozen=True):
+    """Configuration for a reference data object when a URI."""
+
+    type: Literal["uri"]
+    """The object type."""
+    uri: str
+    """The absolute URI of the reference data (as Parquet)."""
+
+    @property
+    def file_extension(self) -> str:
+        """The file extension of the reference uri"""
+        return fh.get_file_suffix(self.uri)  # type: ignore
+
+
+ReferenceConfig = Union[ReferenceFile, ReferenceTable, ReferenceURI]
+"""The config utilised to load the reference data"""
+
+ReferenceConfigUnion = Annotated[ReferenceConfig, Field(discriminator="type")]
+"""Discriminated union to determine refdata config from supplied type"""
+
+
+class BaseRefDataLoader(Generic[EntityType], Mapping[EntityName, EntityType], ABC):
+    """A reference data mapper which lazy-loads requested entities."""
+
+    __entity_type__: ClassVar[type[EntityType]]  # type: ignore
+    """
+    The entity type used for the reference data.
+
+    This will be populated from the generic annotation at class creation time.
+
+    """
+    __step_functions__: ClassVar[dict[type[ReferenceConfig], Callable]] = {}
+    """
+    A mapping between refdata config types and functions to call to load these configs
+    into reference data entities
+    """
+
+    __reader_functions__: ClassVar[dict[str, Callable]] = {}
+    """
+    A mapping between file extensions and functions to load the file uris
+    into reference data entities
+    """
+    prefix: str = "refdata_"
+
+    def __init_subclass__(cls, *_, **__) -> None:
+        """When this class is subclassed, create and populate the `__step_functions__`
+        class variable for the subclass.
+
+        """
+        # Set entity type from parent class subscript.
+        if cls is not BaseRefDataLoader:
+            cls.__entity_type__ = get_entity_type(cls, "BaseRefDataLoader")
+
+        # ensure that dicts are specific to each subclass - redefine rather
+        # than keep the same reference
+        cls.__reader_functions__ = {}
+        cls.__step_functions__ = {}
+
+        for method_name in dir(cls):
+            if method_name.startswith("_"):
+                continue
+
+            method = getattr(cls, method_name, None)
+            if method is None or not callable(method):
+                continue
+
+            if ext := getattr(method, _FILE_EXTENSION_NAME, None):
+                cls.__reader_functions__[ext] = method
+                continue
+
+            type_hints = get_type_hints(method)
+            if set(type_hints.keys()) != {"config", "return"}:
+                continue
+            config_type = type_hints["config"]
+            if not issubclass(config_type, BaseModel):
+                continue
+
+            cls.__step_functions__[config_type] = method  # type: ignore
+
+    # pylint: disable=unused-argument
+    def __init__(
+        self,
+        reference_entity_config: dict[EntityName, ReferenceConfig],
+        dataset_config_uri: Optional[URI] = None,
+        **kwargs,
+    ) -> None:
+        self.reference_entity_config = reference_entity_config
+        self.dataset_config_uri = dataset_config_uri
+        """
+        Configuration options for the reference data. This is likely to vary
+        from backend to backend (e.g. might be locations and file types for
+        some backends, and table names for others).
+
+        """
+        self.entity_cache: dict[EntityName, EntityType] = {}
+        """A cache for already-loaded entities."""
+
+    @abstractmethod
+    def load_table(self, config: ReferenceTable) -> EntityType:
+        """Load reference entity from a database table"""
+        raise NotImplementedError()
+
+    def load_file(self, config: ReferenceFile) -> EntityType:
+        "Load reference entity from a relative file path"
+        if not self.dataset_config_uri:
+            raise AttributeError("dataset_config_uri must be specified if using relative paths")
+        target_location = fh.build_relative_uri(self.dataset_config_uri, config.filename)
+        if isinstance(_get_implementation(self.dataset_config_uri), LocalFilesystemImplementation):
+            target_location = fh.file_uri_to_local_path(target_location).as_posix()
+        try:
+            impl = self.__reader_functions__[config.file_extension]
+            return impl(self, target_location)
+        except KeyError as exc:
+            raise RefdataLacksFileExtensionSupport(file_extension=config.file_extension) from exc
+
+    def load_uri(self, config: ReferenceURI) -> EntityType:
+        "Load reference entity from an absolute URI"
+        if isinstance(_get_implementation(config.uri), LocalFilesystemImplementation):
+            target_location = fh.file_uri_to_local_path(config.uri).as_posix()
+        else:
+            target_location = config.uri
+        try:
+            impl = self.__reader_functions__[config.file_extension]
+            return impl(self, target_location)
+        except KeyError as exc:
+            raise RefdataLacksFileExtensionSupport(file_extension=config.file_extension) from exc
+
+    def load_entity(self, entity_name: EntityName, config: ReferenceConfig) -> EntityType:
+        """Load a reference entity given the reference config"""
+        config_type = type(config)
+        func = self.__step_functions__[config_type]
+        entity = func(self, config)
+        self.entity_cache[entity_name] = entity
+        return entity
+
+    def __getitem__(self, key: EntityName) -> EntityType:
+        try:
+            return self.entity_cache[key]
+        except KeyError:
+            try:
+                config = self.reference_entity_config[key]
+                return self.load_entity(entity_name=key, config=config)
+            except Exception as err:
+                raise MissingRefDataEntity(entity_name=key) from err
+
+    def __iter__(self) -> Iterator[str]:
+        return iter(self.reference_entity_config.keys())
+
+    def __len__(self) -> int:
+        return len(self.reference_entity_config)