tgedr-dataops 0.0.33__py3-none-any.whl → 0.0.36__py3-none-any.whl

tgedr/dataops/source/delta_table_source.py

@@ -3,8 +3,9 @@ import logging
 from typing import Any, Dict, List, Optional
 from pandas import DataFrame
 from deltalake import DeltaTable
+from deltalake.exceptions import TableNotFoundError
 
-from tgedr.dataops.source.source import Source, SourceException
+from tgedr.dataops.source.source import Source, SourceException, NoSourceException
 
 
 logger = logging.getLogger()
@@ -36,10 +37,13 @@ class DeltaTableSource(Source, ABC):
         if self.CONTEXT_KEY_COLUMNS in context:
             columns = context[self.CONTEXT_KEY_COLUMNS]
 
-        delta_table = DeltaTable(
-            table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
-        )
-        result = delta_table.to_pandas(columns=columns)
+        try:
+            delta_table = DeltaTable(
+                table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
+            )
+            result = delta_table.to_pandas(columns=columns)
+        except TableNotFoundError as tnfe:
+            raise NoSourceException(f"could not find delta table: {context[self.CONTEXT_KEY_URL]}")
 
         logger.info(f"[get|out] => {result}")
         return result

tgedr/dataops/source/source.py

@@ -8,6 +8,10 @@ class SourceException(Exception):
     pass
 
 
+class NoSourceException(SourceException):
+    pass
+
+
 class SourceInterface(metaclass=abc.ABCMeta):
     """
     def get(self, context: Optional[Dict[str, Any]] = None) -> Any:

tgedr/dataops/store/s3_single_partition_parquet.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pyarrow as pa
 
 from tgedr.dataops.store.fs_single_partition_parquet import FsSinglePartitionParquetStore
-from src.nn.gs.ss.dataops.commons.utils_fs import remove_s3_protocol
+from tgedr.dataops.commons.utils_fs import remove_s3_protocol
 
 
 logger = logging.getLogger(__name__)

tgedr/dataops/validation/pyspark.py

@@ -0,0 +1,10 @@
+from typing import Any
+from great_expectations.dataset.dataset import Dataset
+
+from tgedr.dataops.validation.abs import DataValidation
+from great_expectations.dataset.sparkdf_dataset import SparkDFDataset
+
+
+class Impl(DataValidation):
+    def _get_dataset(self, df: Any) -> Dataset:
+        return SparkDFDataset(df)

{tgedr_dataops-0.0.33.dist-info → tgedr_dataops-0.0.36.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.33
+Version: 0.0.36
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas

{tgedr_dataops-0.0.33.dist-info → tgedr_dataops-0.0.36.dist-info}/RECORD

@@ -14,23 +14,24 @@ tgedr/dataops/sink/s3_file_sink.py,sha256=m1D6SwuqYEVqvdA9XrL6nVlX_oWLRDT6v74mh_
 tgedr/dataops/sink/sink.py,sha256=8rG3ZNpzeZ82Ac1IoPzkdQTs006IbG-k39APFCeXogk,1271
 tgedr/dataops/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tgedr/dataops/source/abstract_s3_file_source.py,sha256=AMnTWxs57Jni46jz5PvrZTRjd-Ah_TUZEP4Rcx7YziM,1513
-tgedr/dataops/source/delta_table_source.py,sha256=ElZezb6JeYRdkvY8Ryk1EJUvbUHCWRrlblmdUZZBSW4,1410
+tgedr/dataops/source/delta_table_source.py,sha256=tXeJCM-QhssztEpyKGzb6zcM1AlaUwV1Y5d2RHCiOTU,1653
 tgedr/dataops/source/local_delta_table.py,sha256=Z9413skOSsNGfSNeBs-ETVXQJcW4bhS3MYmCwySo9Tc,1545
 tgedr/dataops/source/local_fs_file_source.py,sha256=wihK_wVV6xb8AlkCmZHSRlRmqqm6w9Aq-PI0oOnhn_k,2570
 tgedr/dataops/source/pd_df_s3_source.py,sha256=3Mc0VDGTOmk1HT951l6Qd_bqL0DpPjlfZk0JQjWh3JE,2092
 tgedr/dataops/source/s3_delta_table.py,sha256=d_FoFDNogjZifRQv8V2OmTXbPYymx9QaQcW15uX34pI,2974
 tgedr/dataops/source/s3_file_copy.py,sha256=DbHvstAqi23cywoG6nHpQxvzerrnepApOsv6zsELYNQ,3930
 tgedr/dataops/source/s3_file_source.py,sha256=C8Y0h89p1eBWNGxV4oTNzVLLwZoAoyshHuMlDKKqDA0,4072
-tgedr/dataops/source/source.py,sha256=xm9XNGG-WTo_EGdJ3cfNfSC-nuHeEO2XeKiPIhv4Qoc,1297
+tgedr/dataops/source/source.py,sha256=REeqluMGLMjoDWtdZthzUYkmVeHemSV7t9wjc6eTpJE,1350
 tgedr/dataops/store/fs_single_partition_parquet.py,sha256=CR3406emhxn33jjObnMotXEmZGfh4Iu5Ygv30FvkY6Y,9695
 tgedr/dataops/store/local_fs_single_partition_parquet.py,sha256=N_I96fqxQAp2fWBngoDci3aR1-kcmkWjIVRD0nUi07U,683
-tgedr/dataops/store/s3_single_partition_parquet.py,sha256=JwMhRO9403OLhepeAdbSDNhmM7I4dLoriQfr2IxymHE,3256
+tgedr/dataops/store/s3_single_partition_parquet.py,sha256=2vSyLb-mZ2mAbRhaDoCcsOEnKzQ30AsUYTlqhKQCroc,3249
 tgedr/dataops/store/spark_delta.py,sha256=AHqIKDi9axOKpMJHt4AiBGX8V2mFV_7vNPRpaw37rDY,15101
 tgedr/dataops/store/store.py,sha256=uAuR7MWVdKRaisQ69rFqliLnJrsgpTzbsOh7uPmLSlI,1315
 tgedr/dataops/validation/abs.py,sha256=84HGUuh6k_uG-ON0bauR4lDBTfUeI3GmxOiWsMkTu3E,1521
 tgedr/dataops/validation/pandas.py,sha256=Vfr38f3txbTy098ufPUcRsCgrYu47Rg35upZ9IXcLSk,315
-tgedr_dataops-0.0.33.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
-tgedr_dataops-0.0.33.dist-info/METADATA,sha256=UBD63O5LSsgSWLEyc-c0avh5AXFtl9ADSBE2HPQN91g,607
-tgedr_dataops-0.0.33.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-tgedr_dataops-0.0.33.dist-info/top_level.txt,sha256=acugNvvENatFXbPxKQD9YI5PpzzehbAwyY1keiIkR7I,6
-tgedr_dataops-0.0.33.dist-info/RECORD,,
+tgedr/dataops/validation/pyspark.py,sha256=4OEnA21_vSwB5HjaD5KZdfIjBnTn3KUAqvVGHnY-zNI,317
+tgedr_dataops-0.0.36.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
+tgedr_dataops-0.0.36.dist-info/METADATA,sha256=O_K6YMP__Vgb4Hf4t-34hZYXFRIrvgV6ZkoW3xDseXw,607
+tgedr_dataops-0.0.36.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+tgedr_dataops-0.0.36.dist-info/top_level.txt,sha256=acugNvvENatFXbPxKQD9YI5PpzzehbAwyY1keiIkR7I,6
+tgedr_dataops-0.0.36.dist-info/RECORD,,