tgedr-dataops 0.0.36__py3-none-any.whl → 1.0.1__py3-none-any.whl

tgedr/dataops/validation/abs.py

@@ -1,46 +0,0 @@
-from abc import ABC, abstractmethod
-import logging
-from typing import Any
-
-from great_expectations.core import ExpectationSuite
-from great_expectations.dataset.dataset import Dataset
-
-from tgedr.dataops.utils_reflection import UtilsReflection
-
-logger = logging.getLogger(__name__)
-
-
-class DataValidationException(Exception):
-    pass
-
-
-class DataValidation(ABC):
-    @staticmethod
-    def get_impl(name: str):
-        logger.info(f"[get_impl|in] ({name})")
-        result = None
-        module = ".".join(__name__.split(".")[:-1]) + "." + name.lower()
-        try:
-            result = UtilsReflection.load_subclass_from_module(module, "Impl", DataValidation)()
-        except Exception as x:
-            raise DataValidationException(f"[get_impl] couldn't load implementation for {name}: {x}")
-        logger.info(f"[get_impl|out] ({result})")
-        return result
-
-    @abstractmethod
-    def _get_dataset(self, df: Any) -> Dataset:
-        raise NotImplementedError("DataValidation")
-
-    def validate(self, df: Any, expectations: dict) -> None:
-        logger.info(f"[validate|in] ({df}, {expectations})")
-
-        try:
-            dataset = self._get_dataset(df)
-
-            validation = dataset.validate(ExpectationSuite(**expectations), only_return_failures=True)
-            result = validation.to_json_dict()
-        except Exception as x:
-            raise DataValidationException(f"[validate] failed data expectations", x)
-
-        logger.info(f"[validate|out] => {result['success']}")
-        return result

tgedr/dataops/validation/pandas.py

@@ -1,10 +0,0 @@
-from typing import Any
-from great_expectations.dataset.dataset import Dataset
-
-from tgedr.dataops.validation.abs import DataValidation
-from great_expectations.dataset.sparkdf_dataset import PandasDataset
-
-
-class Impl(DataValidation):
-    def _get_dataset(self, df: Any) -> Dataset:
-        return PandasDataset(df)

tgedr/dataops/validation/pyspark.py

@@ -1,10 +0,0 @@
-from typing import Any
-from great_expectations.dataset.dataset import Dataset
-
-from tgedr.dataops.validation.abs import DataValidation
-from great_expectations.dataset.sparkdf_dataset import SparkDFDataset
-
-
-class Impl(DataValidation):
-    def _get_dataset(self, df: Any) -> Dataset:
-        return SparkDFDataset(df)

tgedr_dataops-0.0.36.dist-info/METADATA

@@ -1,20 +0,0 @@
-Metadata-Version: 2.1
-Name: tgedr-dataops
-Version: 0.0.36
-Summary: data operations related code
-Home-page: https://github.com/jtviegas-sandbox/dataops
-Author: joao tiago viegas
-Author-email: jtviegas@gmail.com
-License: Unlicense
-Keywords: data engineering mlops ml
-Classifier: Development Status :: 3 - Alpha
-Classifier: Programming Language :: Python :: 3.10
-Requires-Python: >=3.9
-License-File: LICENSE
-Requires-Dist: pandas ==1.5.3
-Requires-Dist: pyarrow ==15.*
-Requires-Dist: s3fs ==2024.5.0
-Requires-Dist: boto3 ==1.34.106
-Requires-Dist: great-expectations ==0.18.10
-Requires-Dist: deltalake ~=0.16.4
-

tgedr_dataops-0.0.36.dist-info/RECORD

@@ -1,37 +0,0 @@
-tgedr/dataops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tgedr/dataops/chain.py,sha256=bo9HhfV3eJmss52HaR8LkV7RYSTKmya0lbcqiovOnQU,1405
-tgedr/dataops/etl.py,sha256=BEnpcBPJxAIXVhwnprTMl-onQHuUn3amuAgPVPJnnhs,3250
-tgedr/dataops/processor.py,sha256=GZmq7yefq6ySgRkxJyHxXUsUhfa6NiHmn1W3SECXVBA,710
-tgedr/dataops/utils_reflection.py,sha256=W3eqrypMwqRsvevEZ9d6Zxw_GJL4d938lXiU1gIOwlo,4841
-tgedr/dataops/commons/dataset.py,sha256=VIt385Zps78y6cbe54G4PLNuUA647S94UqokubM1gKI,637
-tgedr/dataops/commons/metadata.py,sha256=5uIurGkkREJuAlONav8_TyLVNY097Zix6LIdRxVE_uQ,5796
-tgedr/dataops/commons/s3_connector.py,sha256=AmD8l86ORbABamkquD8BMtMpgSfThkVbl9Kf54wR_pA,1194
-tgedr/dataops/commons/utils_fs.py,sha256=8gXPcdQx520pr1h7tn7m_Z1AojgWIdebNLCyiQgEZEU,2362
-tgedr/dataops/commons/utils_spark.py,sha256=VSFUcQt-yE54VHJkonveFwhM3ADTK3kfC3QfA34QOVI,3191
-tgedr/dataops/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tgedr/dataops/sink/local_fs_file_sink.py,sha256=JjFWwW_ufWF8I-PVPP1TleKmRj8Des-6A0xLqtbPYW8,1508
-tgedr/dataops/sink/s3_file_sink.py,sha256=m1D6SwuqYEVqvdA9XrL6nVlX_oWLRDT6v74mh_diuQM,2362
-tgedr/dataops/sink/sink.py,sha256=8rG3ZNpzeZ82Ac1IoPzkdQTs006IbG-k39APFCeXogk,1271
-tgedr/dataops/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tgedr/dataops/source/abstract_s3_file_source.py,sha256=AMnTWxs57Jni46jz5PvrZTRjd-Ah_TUZEP4Rcx7YziM,1513
-tgedr/dataops/source/delta_table_source.py,sha256=tXeJCM-QhssztEpyKGzb6zcM1AlaUwV1Y5d2RHCiOTU,1653
-tgedr/dataops/source/local_delta_table.py,sha256=Z9413skOSsNGfSNeBs-ETVXQJcW4bhS3MYmCwySo9Tc,1545
-tgedr/dataops/source/local_fs_file_source.py,sha256=wihK_wVV6xb8AlkCmZHSRlRmqqm6w9Aq-PI0oOnhn_k,2570
-tgedr/dataops/source/pd_df_s3_source.py,sha256=3Mc0VDGTOmk1HT951l6Qd_bqL0DpPjlfZk0JQjWh3JE,2092
-tgedr/dataops/source/s3_delta_table.py,sha256=d_FoFDNogjZifRQv8V2OmTXbPYymx9QaQcW15uX34pI,2974
-tgedr/dataops/source/s3_file_copy.py,sha256=DbHvstAqi23cywoG6nHpQxvzerrnepApOsv6zsELYNQ,3930
-tgedr/dataops/source/s3_file_source.py,sha256=C8Y0h89p1eBWNGxV4oTNzVLLwZoAoyshHuMlDKKqDA0,4072
-tgedr/dataops/source/source.py,sha256=REeqluMGLMjoDWtdZthzUYkmVeHemSV7t9wjc6eTpJE,1350
-tgedr/dataops/store/fs_single_partition_parquet.py,sha256=CR3406emhxn33jjObnMotXEmZGfh4Iu5Ygv30FvkY6Y,9695
-tgedr/dataops/store/local_fs_single_partition_parquet.py,sha256=N_I96fqxQAp2fWBngoDci3aR1-kcmkWjIVRD0nUi07U,683
-tgedr/dataops/store/s3_single_partition_parquet.py,sha256=2vSyLb-mZ2mAbRhaDoCcsOEnKzQ30AsUYTlqhKQCroc,3249
-tgedr/dataops/store/spark_delta.py,sha256=AHqIKDi9axOKpMJHt4AiBGX8V2mFV_7vNPRpaw37rDY,15101
-tgedr/dataops/store/store.py,sha256=uAuR7MWVdKRaisQ69rFqliLnJrsgpTzbsOh7uPmLSlI,1315
-tgedr/dataops/validation/abs.py,sha256=84HGUuh6k_uG-ON0bauR4lDBTfUeI3GmxOiWsMkTu3E,1521
-tgedr/dataops/validation/pandas.py,sha256=Vfr38f3txbTy098ufPUcRsCgrYu47Rg35upZ9IXcLSk,315
-tgedr/dataops/validation/pyspark.py,sha256=4OEnA21_vSwB5HjaD5KZdfIjBnTn3KUAqvVGHnY-zNI,317
-tgedr_dataops-0.0.36.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
-tgedr_dataops-0.0.36.dist-info/METADATA,sha256=O_K6YMP__Vgb4Hf4t-34hZYXFRIrvgV6ZkoW3xDseXw,607
-tgedr_dataops-0.0.36.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-tgedr_dataops-0.0.36.dist-info/top_level.txt,sha256=acugNvvENatFXbPxKQD9YI5PpzzehbAwyY1keiIkR7I,6
-tgedr_dataops-0.0.36.dist-info/RECORD,,

tgedr_dataops-0.0.36.dist-info/top_level.txt

@@ -1 +0,0 @@
-tgedr