data-validation-engine 0.6.2__py3-none-any.whl

dve/parser/file_handling/utilities.py

@@ -0,0 +1,53 @@
+"""Utilities for working with files."""
+
+import tempfile
+from pathlib import Path
+from types import TracebackType
+from typing import Optional
+
+from dve.parser.exceptions import UnsupportedSchemeError
+from dve.parser.file_handling.service import is_supported, remove_prefix
+from dve.parser.type_hints import URI
+
+
+class TemporaryPrefix:
+    """Like 'TemporaryDirectory', but with support for a URL prefix."""
+
+    def __init__(self, prefix: Optional[URI] = None):
+        """Set up the prefix.
+
+        Args:
+         - `prefix`: the URL prefix to use as temporary storage. This
+           will default to a local temporary folder.
+
+        """
+        if not prefix:
+            prefix = Path(tempfile.mkdtemp()).as_uri()
+        self._prefix = prefix.rstrip("/") + "/"
+
+        # Ensure we have an implementation for this prefix.
+        if not is_supported(self._prefix):  # pragma: no cover
+            raise UnsupportedSchemeError(f"No supported implementation for {prefix!r}")
+        self._in_context = False
+
+    @property
+    def prefix(self) -> URI:  # pragma: no cover
+        """The URI prefix of the temporary directory."""
+        if not self._in_context:
+            raise ValueError(f"`{self.__class__.__name__}` must be used as context manager")
+        return self._prefix
+
+    def __enter__(self) -> URI:
+        """Enters the context manager and yields the prefix"""
+        self._in_context = True
+        return self._prefix
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[Exception]],
+        exc_value: Optional[Exception],
+        traceback: Optional[TracebackType],
+    ):
+        """Exits the context manager and cleans up the temporary prefix"""
+        self._in_context = False
+        remove_prefix(self._prefix, recursive=True)

dve/parser/type_hints.py

@@ -0,0 +1,46 @@
+"""Type hints for the parser."""
+
+from pathlib import Path
+from typing import Any, Optional, Union
+
+from typing_extensions import Literal
+
+PathStr = str
+"""A filesystem path, as a string (cursed)."""
+URI = str
+"""A URI representing a remote or local resource."""
+Filename = str
+"""A string representing a filename."""
+Scheme = str
+"""The scheme attribute of the URI."""
+Hostname = Optional[str]
+"""The hostname attribute of the URI."""
+URIPath = str
+"""The path attribute of the URI."""
+Extension = str
+"""A file extension (e.g. '.csv')."""
+TextFileOpenMode = Literal["r", "a", "w", "a+"]
+"""An opening mode for a file in text mode."""
+BinaryFileOpenMode = Literal["ab", "rb", "wb", "ba", "br", "bw"]
+"""An opening mode for a file in binary mode."""
+FileOpenMode = Union[TextFileOpenMode, BinaryFileOpenMode]
+"""An opening mode for a file."""
+NodeType = Literal["resource", "directory"]
+"""The type of node in a filesystem."""
+
+Location = Union[PathStr, Path, URI]
+"""
+A filesystem or remote location. An annoying, difficult to resolve union
+(see `parser.file_handling.service.resolve_location`).
+
+"""
+
+ReaderName = str
+"""A parser name. This must be importable from `parser.readers`"""
+ReaderArgs = Optional[dict[str, Any]]
+"""Keyword arguments to be passed to the parser's constructor."""
+FieldName = str
+"""The name of a field within the dataset."""
+
+SparkXMLMode = Literal["PERMISSIVE", "FAILFAST", "DROPMALFORMED"]
+"""The mode to use when parsing XML files with Spark."""

dve/parser/utilities.py

@@ -0,0 +1,113 @@
+"""Useful helper functions.
+
+This is functionality which may be useful for multiple readers,
+but isn't quite universal enough to be in e.g. a base class.
+
+"""
+
+from collections import defaultdict
+from collections.abc import Iterable, Iterator
+from itertools import tee
+from typing import TypeVar, Union, overload
+
+from pyspark.sql.types import ArrayType, StringType, StructField, StructType
+
+T = TypeVar("T")
+TemplateElement = Union[None, list["TemplateElement"], dict[str, "TemplateElement"]]  # type: ignore
+"""The base types used in the template row."""
+TemplateRow = dict[str, "TemplateElement"]  # type: ignore
+"""The type of a template row."""
+
+
+def peek(iterable: Iterable[T]) -> tuple[T, Iterator[T]]:
+    """Peek the first item from an iterable, returning the first item
+    and an iterator representing the state of the iterable _before_
+    the first item was taken.
+
+    """
+    current, clone = tee(iterable, 2)
+    return next(clone), current
+
+
+@overload
+def template_row_to_spark_schema(template_element: TemplateRow) -> StructType: ...
+
+
+@overload
+def template_row_to_spark_schema(
+    template_element: TemplateElement,
+) -> Union[ArrayType, StringType, StructType]: ...
+
+
+def template_row_to_spark_schema(template_element):
+    """Get a Spark schema from a template row."""
+    # Should we implement the full logic from dve.core_engine.spark_helpers here?
+    if template_element is None:
+        return StringType()
+    if isinstance(template_element, list):
+        if not template_element:
+            nested_type = None
+        elif len(template_element) == 1:
+            nested_type = template_element[0]
+        else:
+            raise ValueError(f"Nested array longer than 1: {template_element!r}")
+        return ArrayType(template_row_to_spark_schema(nested_type))
+    if not isinstance(template_element, dict):
+        raise TypeError(f"Must be dict, list, or None, got {template_element!r}")
+
+    fields = []
+    for field_name, nested_type in template_element.items():
+        fields.append(StructField(str(field_name), template_row_to_spark_schema(nested_type)))
+    return StructType(fields)
+
+
+def parse_template_row(field_names: Iterable[str]) -> TemplateRow:
+    """Parse a template row.
+
+    Field names can be separated by level using '.', and wrapping the
+    field name in square brackets indicates that the item is expected
+    to be an array.
+
+    >>> parse_template_row(['name'])
+    {'name': None}
+    >>> parse_template_row(['[name]'])
+    {'name': [None]}
+    >>> parse_template_row(['name', 'name.nested'])
+    {'name': {'nested': None}}
+    >>> parse_template_row(['[name]', 'name.nested'])
+    {'name': [{'nested': None}]}
+    >>> parse_template_row(['name', '[name.nested_list]'])
+    {'name': {'nested_list': [None]}}
+    >>> parse_template_row(['[name]', '[name.nested_list]'])
+    {'name': [{'nested_list': [None]}]}
+
+    """
+    array_levels = set()
+    sub_levels_by_level: dict[str, list[str]] = defaultdict(list)
+
+    for name in field_names:
+        is_array = name.startswith("[")
+        name = name.strip("[]")
+
+        if "." not in name:
+            # Add the key to the defaultdict, if it's not already added
+            sub_levels_by_level[name]  # pylint: disable=pointless-statement
+            if is_array:
+                array_levels.add(name)
+        else:
+            level, sub_level = name.split(".", 1)
+            if is_array:
+                sub_level = f"[{sub_level}]"
+            sub_levels_by_level[level].append(sub_level)
+
+    row = {}
+    for level, sub_level_names in sub_levels_by_level.items():
+        value: TemplateElement = None
+        if sub_level_names:
+            value = parse_template_row(sub_level_names)
+
+        if level in array_levels:
+            value = [value]
+        row[level] = value
+
+    return row

dve/pipeline/duckdb_pipeline.py

@@ -0,0 +1,56 @@
+"""DuckDB implementation for `Pipeline` object."""
+
+import logging
+from typing import Optional
+
+from duckdb import DuckDBPyConnection, DuckDBPyRelation
+
+from dve.core_engine.backends.base.reference_data import BaseRefDataLoader
+from dve.core_engine.backends.implementations.duckdb.auditing import DDBAuditingManager
+from dve.core_engine.backends.implementations.duckdb.contract import DuckDBDataContract
+from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import duckdb_get_entity_count
+from dve.core_engine.backends.implementations.duckdb.rules import DuckDBStepImplementations
+from dve.core_engine.models import SubmissionInfo
+from dve.core_engine.type_hints import URI
+from dve.pipeline.pipeline import BaseDVEPipeline
+
+
+# pylint: disable=abstract-method
+@duckdb_get_entity_count
+class DDBDVEPipeline(BaseDVEPipeline):
+    """
+    Modified Pipeline class for running a DVE Pipeline with Spark
+    """
+
+    # pylint: disable=R0913
+    def __init__(
+        self,
+        processed_files_path: URI,
+        audit_tables: DDBAuditingManager,
+        connection: DuckDBPyConnection,
+        rules_path: Optional[URI],
+        submitted_files_path: Optional[URI],
+        reference_data_loader: Optional[type[BaseRefDataLoader]] = None,
+        job_run_id: Optional[int] = None,
+        logger: Optional[logging.Logger] = None,
+    ):
+        self._connection = connection
+        super().__init__(
+            processed_files_path,
+            audit_tables,
+            DuckDBDataContract(connection=self._connection),
+            DuckDBStepImplementations.register_udfs(connection=self._connection),
+            rules_path,
+            submitted_files_path,
+            reference_data_loader,
+            job_run_id,
+            logger,
+        )
+
+    # pylint: disable=arguments-differ
+    def write_file_to_parquet(  # type: ignore
+        self, submission_file_uri: URI, submission_info: SubmissionInfo, output: URI
+    ):
+        return super().write_file_to_parquet(
+            submission_file_uri, submission_info, output, DuckDBPyRelation
+        )

dve/pipeline/foundry_ddb_pipeline.py

@@ -0,0 +1,171 @@
+# pylint: disable=W0223
+"""A duckdb pipeline for running on Foundry platform"""
+
+import shutil
+from pathlib import Path
+from typing import Optional
+
+from dve.common.error_utils import dump_processing_errors
+from dve.core_engine.backends.implementations.duckdb.duckdb_helpers import (
+    duckdb_get_entity_count,
+    duckdb_write_parquet,
+)
+from dve.core_engine.exceptions import CriticalProcessingError
+from dve.core_engine.models import SubmissionInfo
+from dve.core_engine.type_hints import URI
+from dve.parser import file_handling as fh
+from dve.parser.file_handling.implementations.file import LocalFilesystemImplementation
+from dve.parser.file_handling.service import _get_implementation
+from dve.pipeline.duckdb_pipeline import DDBDVEPipeline
+from dve.pipeline.utils import SubmissionStatus
+
+
+@duckdb_get_entity_count
+@duckdb_write_parquet
+class FoundryDDBPipeline(DDBDVEPipeline):
+    """DuckDB pipeline for running on Foundry Platform"""
+
+    def _move_submission_to_processing_files_path(self, submission_info: SubmissionInfo):
+        """Move submitted file to 'processed_files_path'."""
+        _submitted_file_location = Path(
+            self._submitted_files_path, submission_info.file_name_with_ext  # type: ignore
+        )
+        _dest = Path(self.processed_files_path, submission_info.submission_id)
+        _dest.mkdir(parents=True, exist_ok=True)
+        shutil.copy2(_submitted_file_location, _dest)
+
+    def persist_audit_records(self, submission_info: SubmissionInfo) -> URI:
+        """Write out key audit relations to parquet for persisting to datasets"""
+        write_to = fh.joinuri(self.processed_files_path, submission_info.submission_id, "audit/")
+        if isinstance(_get_implementation(write_to), LocalFilesystemImplementation):
+            write_to = fh.file_uri_to_local_path(write_to)
+            write_to.parent.mkdir(parents=True, exist_ok=True)
+            write_to = write_to.as_posix()
+        self.write_parquet(  # type: ignore # pylint: disable=E1101
+            self._audit_tables._processing_status.get_relation(),  # pylint: disable=W0212
+            fh.joinuri(write_to, "processing_status.parquet"),
+        )
+        self.write_parquet(  # type: ignore # pylint: disable=E1101
+            self._audit_tables._submission_statistics.get_relation(),  # pylint: disable=W0212
+            fh.joinuri(write_to, "submission_statistics.parquet"),
+        )
+        return write_to
+
+    def file_transformation(
+        self, submission_info: SubmissionInfo
+    ) -> tuple[SubmissionInfo, SubmissionStatus]:
+        try:
+            return super().file_transformation(submission_info)
+        except Exception as exc:  # pylint: disable=W0718
+            self._logger.exception("File transformation raised exception:")
+            dump_processing_errors(
+                fh.joinuri(self.processed_files_path, submission_info.submission_id),
+                "file_transformation",
+                [CriticalProcessingError.from_exception(exc)],
+            )
+            self._audit_tables.mark_failed(submissions=[submission_info.submission_id])
+            return submission_info, SubmissionStatus(processing_failed=True)
+
+    def apply_data_contract(
+        self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None
+    ) -> tuple[SubmissionInfo, SubmissionStatus]:
+        try:
+            return super().apply_data_contract(submission_info, submission_status)
+        except Exception as exc:  # pylint: disable=W0718
+            self._logger.exception("Apply data contract raised exception:")
+            dump_processing_errors(
+                fh.joinuri(self.processed_files_path, submission_info.submission_id),
+                "data_contract",
+                [CriticalProcessingError.from_exception(exc)],
+            )
+            self._audit_tables.mark_failed(submissions=[submission_info.submission_id])
+            return submission_info, SubmissionStatus(processing_failed=True)
+
+    def apply_business_rules(
+        self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None
+    ):
+        try:
+            return super().apply_business_rules(submission_info, submission_status)
+        except Exception as exc:  # pylint: disable=W0718
+            self._logger.exception("Apply business rules raised exception:")
+            dump_processing_errors(
+                fh.joinuri(self.processed_files_path, submission_info.submission_id),
+                "business_rules",
+                [CriticalProcessingError.from_exception(exc)],
+            )
+            self._audit_tables.mark_failed(submissions=[submission_info.submission_id])
+            return submission_info, SubmissionStatus(processing_failed=True)
+
+    def error_report(
+        self, submission_info: SubmissionInfo, submission_status: Optional[SubmissionStatus] = None
+    ):
+        try:
+            return super().error_report(submission_info, submission_status)
+        except Exception as exc:  # pylint: disable=W0718
+            self._logger.exception("Error reports raised exception:")
+            sub_stats = None
+            report_uri = None
+            submission_status = submission_status if submission_status else SubmissionStatus()
+            submission_status.processing_failed = True
+            dump_processing_errors(
+                fh.joinuri(self.processed_files_path, submission_info.submission_id),
+                "error_report",
+                [CriticalProcessingError.from_exception(exc)],
+            )
+            self._audit_tables.mark_failed(submissions=[submission_info.submission_id])
+            return submission_info, submission_status, sub_stats, report_uri
+
+    def run_pipeline(
+        self, submission_info: SubmissionInfo
+    ) -> tuple[Optional[URI], Optional[URI], URI]:
+        """Sequential single submission pipeline runner"""
+        try:
+            sub_id: str = submission_info.submission_id
+            report_uri = None
+            if self._submitted_files_path:
+                self._move_submission_to_processing_files_path(submission_info)
+            self._audit_tables.add_new_submissions(submissions=[submission_info])
+            self._audit_tables.mark_transform(submission_ids=[sub_id])
+            sub_info, sub_status = self.file_transformation(submission_info=submission_info)
+            if not (sub_status.validation_failed or sub_status.processing_failed):
+                self._audit_tables.mark_data_contract(submission_ids=[sub_id])
+                sub_info, sub_status = self.apply_data_contract(
+                    submission_info=sub_info, submission_status=sub_status
+                )
+                self._audit_tables.mark_business_rules(
+                    submissions=[(sub_id, sub_status.validation_failed)]
+                )
+                sub_info, sub_status = self.apply_business_rules(
+                    submission_info=submission_info, submission_status=sub_status
+                )
+
+            if not sub_status.processing_failed:
+                self._audit_tables.mark_error_report(
+                    submissions=[(sub_id, sub_status.submission_result)]
+                )
+                sub_info, sub_status, sub_stats, report_uri = self.error_report(
+                    submission_info=submission_info, submission_status=sub_status
+                )
+                if sub_stats:
+                    self._audit_tables.add_submission_statistics_records(sub_stats=[sub_stats])
+        except Exception as err:  # pylint: disable=W0718
+            self._logger.exception(
+                f"During processing of submission_id: {sub_id}, this exception was raised:"
+            )
+            dump_processing_errors(
+                fh.joinuri(self.processed_files_path, submission_info.submission_id),
+                "pipeline",
+                [CriticalProcessingError.from_exception(err)],
+            )
+            self._audit_tables.mark_failed(submissions=[sub_id])
+        finally:
+            audit_files_uri = self.persist_audit_records(submission_info=submission_info)
+        return (
+            (
+                None
+                if (sub_status.validation_failed or sub_status.processing_failed)
+                else fh.joinuri(self.processed_files_path, sub_id, "business_rules")
+            ),
+            report_uri if report_uri else None,
+            audit_files_uri,
+        )