tgedr-dataops 0.0.33__tar.gz → 0.0.36__tar.gz

{tgedr_dataops-0.0.33/src/tgedr_dataops.egg-info → tgedr_dataops-0.0.36}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.33
+Version: 0.0.36
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas

{tgedr_dataops-0.0.33 → tgedr_dataops-0.0.36}/README.md

@@ -42,8 +42,8 @@ data operations related code
 
 #### validation
 - __DataValidation__ : abstract class defining a `validate` method to perform data validation, currently using Great Expectations library
-- __pandas.Impl__ : __DataValidation__ implementation to validate pandas dataframes with Great Expectations library ([example](test/tgedr/dataops/validation/test_validation.py))
-
+- __pandas.Impl__ : __DataValidation__ implementation to validate pandas dataframes with Great Expectations library ([example](test/tgedr/dataops/validation/test_pandas.py))
+- __pyspark.Impl__ : __DataValidation__ implementation to validate pyspark dataframes with Great Expectations library ([example](test/tgedr/dataops/validation/test_pyspark.py))
 
 
 ## installation

{tgedr_dataops-0.0.33 → tgedr_dataops-0.0.36}/setup.py

@@ -4,7 +4,7 @@ import os
 from setuptools import setup, find_namespace_packages
 
 logger = logging.getLogger(__name__)
-VERSION = "0.0.33"
+VERSION = "0.0.36"
 logging.info(f"building version: {VERSION}")
 
 setup(

{tgedr_dataops-0.0.33 → tgedr_dataops-0.0.36}/src/tgedr/dataops/source/delta_table_source.py

@@ -3,8 +3,9 @@ import logging
 from typing import Any, Dict, List, Optional
 from pandas import DataFrame
 from deltalake import DeltaTable
+from deltalake.exceptions import TableNotFoundError
 
-from tgedr.dataops.source.source import Source, SourceException
+from tgedr.dataops.source.source import Source, SourceException, NoSourceException
 
 
 logger = logging.getLogger()
@@ -36,10 +37,13 @@ class DeltaTableSource(Source, ABC):
         if self.CONTEXT_KEY_COLUMNS in context:
             columns = context[self.CONTEXT_KEY_COLUMNS]
 
-        delta_table = DeltaTable(
-            table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
-        )
-        result = delta_table.to_pandas(columns=columns)
+        try:
+            delta_table = DeltaTable(
+                table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
+            )
+            result = delta_table.to_pandas(columns=columns)
+        except TableNotFoundError as tnfe:
+            raise NoSourceException(f"could not find delta table: {context[self.CONTEXT_KEY_URL]}")
 
         logger.info(f"[get|out] => {result}")
         return result

{tgedr_dataops-0.0.33 → tgedr_dataops-0.0.36}/src/tgedr/dataops/source/source.py

@@ -8,6 +8,10 @@ class SourceException(Exception):
     pass
 
 
+class NoSourceException(SourceException):
+    pass
+
+
 class SourceInterface(metaclass=abc.ABCMeta):
     """
     def get(self, context: Optional[Dict[str, Any]] = None) -> Any:

{tgedr_dataops-0.0.33 → tgedr_dataops-0.0.36}/src/tgedr/dataops/store/s3_single_partition_parquet.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pyarrow as pa
 
 from tgedr.dataops.store.fs_single_partition_parquet import FsSinglePartitionParquetStore
-from src.nn.gs.ss.dataops.commons.utils_fs import remove_s3_protocol
+from tgedr.dataops.commons.utils_fs import remove_s3_protocol
 
 
 logger = logging.getLogger(__name__)

tgedr_dataops-0.0.36/src/tgedr/dataops/validation/pyspark.py

@@ -0,0 +1,10 @@
+from typing import Any
+from great_expectations.dataset.dataset import Dataset
+
+from tgedr.dataops.validation.abs import DataValidation
+from great_expectations.dataset.sparkdf_dataset import SparkDFDataset
+
+
+class Impl(DataValidation):
+    def _get_dataset(self, df: Any) -> Dataset:
+        return SparkDFDataset(df)

{tgedr_dataops-0.0.33 → tgedr_dataops-0.0.36/src/tgedr_dataops.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.33
+Version: 0.0.36
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas

{tgedr_dataops-0.0.33 → tgedr_dataops-0.0.36}/src/tgedr_dataops.egg-info/SOURCES.txt

@@ -33,6 +33,7 @@ src/tgedr/dataops/store/spark_delta.py
 src/tgedr/dataops/store/store.py
 src/tgedr/dataops/validation/abs.py
 src/tgedr/dataops/validation/pandas.py
+src/tgedr/dataops/validation/pyspark.py
 src/tgedr_dataops.egg-info/PKG-INFO
 src/tgedr_dataops.egg-info/SOURCES.txt
 src/tgedr_dataops.egg-info/dependency_links.txt