tgedr-dataops 0.0.34__py3-none-any.whl → 0.0.37__py3-none-any.whl

tgedr/dataops/source/pd_df_s3_source.py

@@ -19,6 +19,7 @@ class PdDfS3Source(AbstractS3FileSource):
     CONTEXT_KEY_COLUMN_NAMES = "column_names"
     CONTEXT_KEY_SCHEMA_TYPES = "schema_types"
     DEFAULT_FORMAT = "csv"
+    FORMATS = ["csv", "xlsx"]
     DEFAULT_SEPARATOR = ","
 
     def __init__(self, config: Optional[Dict[str, Any]] = None):
@@ -33,6 +34,23 @@ class PdDfS3Source(AbstractS3FileSource):
         if self.CONTEXT_KEY_URL not in context:
             raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
 
+        format: str = self.DEFAULT_FORMAT
+        if self.CONTEXT_KEY_FILE_FORMAT in context:
+            format = context[self.CONTEXT_KEY_FILE_FORMAT]
+            if format not in self.FORMATS:
+                raise SourceException(f"[get] invalid format: {format}")
+
+        if "csv" == format:
+            result = self.__read_csv(context=context)
+        else:
+            result = self.__read_excel(context=context)
+
+        logger.info(f"[get|out] => {result}")
+        return result
+
+    def __read_csv(self, context: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        logger.info(f"[__read_csv|in] ({context})")
+
         protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
 
         obj = self._client.get_object(Bucket=bucket, Key=key)
@@ -45,7 +63,14 @@ class PdDfS3Source(AbstractS3FileSource):
             self.DEFAULT_SEPARATOR if self.CONTEXT_KEY_SEPARATOR not in context else context[self.CONTEXT_KEY_SEPARATOR]
         )
 
-        result = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
+        result: pd.DataFrame = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
 
-        logger.info(f"[get|out] => {result}")
+        logger.info(f"[__read_csv|out] => {result}")
+        return result
+
+    def __read_excel(self, context: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        logger.info(f"[__read_excel|in] ({context})")
+        src = context[self.CONTEXT_KEY_URL]
+        result: pd.DataFrame = pd.read_excel(src, engine="openpyxl")
+        logger.info(f"[__read_excel|out] => {result}")
         return result

tgedr/dataops/source/s3_file_extended_source.py

@@ -0,0 +1,39 @@
+import logging
+from typing import Any, Dict, Optional
+
+from tgedr.dataops.source.s3_file_source import S3FileSource
+from tgedr.dataops.source.source import SourceException
+from tgedr.dataops.commons.utils_fs import process_s3_path
+
+
+logger = logging.getLogger(__name__)
+
+
+class S3FileExtendedSource(S3FileSource):
+    """class used to retrieve objects/files from s3 bucket to local fs location"""
+
+    METADATA_KEYS = ["LastModified", "ContentLength", "ETag", "VersionId", "ContentType"]
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config=config)
+
+    def get_metadata(self, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        logger.info(f"[get_metadata|in] ({context})")
+
+        result: Dict[str, Any] = {}
+        if self.CONTEXT_KEY_SOURCE not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_SOURCE}")
+
+        bucket, key = process_s3_path(context[self.CONTEXT_KEY_SOURCE])
+
+        o = self._client.head_object(Bucket=bucket, Key=key)
+
+        for key in list(o.keys()):
+            if key in self.METADATA_KEYS:
+                if key == "LastModified":
+                    result[key] = int(o[key].timestamp())
+                else:
+                    result[key] = o[key]
+
+        logger.info(f"[get_metadata|out] => result len: {result}")
+        return result

tgedr/dataops/source/s3_file_source.py

@@ -22,21 +22,6 @@ class S3FileSource(Source, S3Connector):
         Source.__init__(self, config=config)
         S3Connector.__init__(self)
 
-    def __derive_local_file(self, target: str, isdir: bool, file: str):
-        logger.debug(f"[__derive_local_file|in] ({target}, {isdir}, {file})")
-
-        if isdir:
-            result = os.path.join(target, file)
-            basedir = os.path.dirname(result)
-            if not os.path.exists(basedir):
-                logger.debug(f"[__derive_local_file] creating folder: {basedir}")
-                os.mkdir(basedir)
-        else:
-            result = target
-
-        logger.debug(f"[__derive_local_file|out] => {result}")
-        return result
-
     def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
         logger.info(f"[list|in] ({context})")
 

tgedr/dataops/store/s3_single_partition_parquet.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pyarrow as pa
 
 from tgedr.dataops.store.fs_single_partition_parquet import FsSinglePartitionParquetStore
-from src.nn.gs.ss.dataops.commons.utils_fs import remove_s3_protocol
+from tgedr.dataops.commons.utils_fs import remove_s3_protocol
 
 
 logger = logging.getLogger(__name__)

tgedr/dataops/validation/pyspark.py

@@ -0,0 +1,10 @@
+from typing import Any
+from great_expectations.dataset.dataset import Dataset
+
+from tgedr.dataops.validation.abs import DataValidation
+from great_expectations.dataset.sparkdf_dataset import SparkDFDataset
+
+
+class Impl(DataValidation):
+    def _get_dataset(self, df: Any) -> Dataset:
+        return SparkDFDataset(df)

{tgedr_dataops-0.0.34.dist-info → tgedr_dataops-0.0.37.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.34
+Version: 0.0.37
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas
@@ -17,4 +17,5 @@ Requires-Dist: s3fs ==2024.5.0
 Requires-Dist: boto3 ==1.34.106
 Requires-Dist: great-expectations ==0.18.10
 Requires-Dist: deltalake ~=0.16.4
+Requires-Dist: openpyxl ==3.1.2
 

{tgedr_dataops-0.0.34.dist-info → tgedr_dataops-0.0.37.dist-info}/RECORD

@@ -17,20 +17,22 @@ tgedr/dataops/source/abstract_s3_file_source.py,sha256=AMnTWxs57Jni46jz5PvrZTRjd
 tgedr/dataops/source/delta_table_source.py,sha256=tXeJCM-QhssztEpyKGzb6zcM1AlaUwV1Y5d2RHCiOTU,1653
 tgedr/dataops/source/local_delta_table.py,sha256=Z9413skOSsNGfSNeBs-ETVXQJcW4bhS3MYmCwySo9Tc,1545
 tgedr/dataops/source/local_fs_file_source.py,sha256=wihK_wVV6xb8AlkCmZHSRlRmqqm6w9Aq-PI0oOnhn_k,2570
-tgedr/dataops/source/pd_df_s3_source.py,sha256=3Mc0VDGTOmk1HT951l6Qd_bqL0DpPjlfZk0JQjWh3JE,2092
+tgedr/dataops/source/pd_df_s3_source.py,sha256=Bfnj363MZtmMJB4W32DKEaYYG0v7NeG-m66C5i8bxsc,3103
 tgedr/dataops/source/s3_delta_table.py,sha256=d_FoFDNogjZifRQv8V2OmTXbPYymx9QaQcW15uX34pI,2974
 tgedr/dataops/source/s3_file_copy.py,sha256=DbHvstAqi23cywoG6nHpQxvzerrnepApOsv6zsELYNQ,3930
-tgedr/dataops/source/s3_file_source.py,sha256=C8Y0h89p1eBWNGxV4oTNzVLLwZoAoyshHuMlDKKqDA0,4072
+tgedr/dataops/source/s3_file_extended_source.py,sha256=B6m84DWV45E1V3XsLkHARp9YVNvmsDi54ikd2__9q5Q,1384
+tgedr/dataops/source/s3_file_source.py,sha256=ayQ-gPxARFc1xINFOM0_hQFUn6odpts_uFaDZYs7RC4,3520
 tgedr/dataops/source/source.py,sha256=REeqluMGLMjoDWtdZthzUYkmVeHemSV7t9wjc6eTpJE,1350
 tgedr/dataops/store/fs_single_partition_parquet.py,sha256=CR3406emhxn33jjObnMotXEmZGfh4Iu5Ygv30FvkY6Y,9695
 tgedr/dataops/store/local_fs_single_partition_parquet.py,sha256=N_I96fqxQAp2fWBngoDci3aR1-kcmkWjIVRD0nUi07U,683
-tgedr/dataops/store/s3_single_partition_parquet.py,sha256=JwMhRO9403OLhepeAdbSDNhmM7I4dLoriQfr2IxymHE,3256
+tgedr/dataops/store/s3_single_partition_parquet.py,sha256=2vSyLb-mZ2mAbRhaDoCcsOEnKzQ30AsUYTlqhKQCroc,3249
 tgedr/dataops/store/spark_delta.py,sha256=AHqIKDi9axOKpMJHt4AiBGX8V2mFV_7vNPRpaw37rDY,15101
 tgedr/dataops/store/store.py,sha256=uAuR7MWVdKRaisQ69rFqliLnJrsgpTzbsOh7uPmLSlI,1315
 tgedr/dataops/validation/abs.py,sha256=84HGUuh6k_uG-ON0bauR4lDBTfUeI3GmxOiWsMkTu3E,1521
 tgedr/dataops/validation/pandas.py,sha256=Vfr38f3txbTy098ufPUcRsCgrYu47Rg35upZ9IXcLSk,315
-tgedr_dataops-0.0.34.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
-tgedr_dataops-0.0.34.dist-info/METADATA,sha256=ryzlSTJx7GtIQNpX3PL2QiZ0h-bsR_smUL-l_rdfXdw,607
-tgedr_dataops-0.0.34.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-tgedr_dataops-0.0.34.dist-info/top_level.txt,sha256=acugNvvENatFXbPxKQD9YI5PpzzehbAwyY1keiIkR7I,6
-tgedr_dataops-0.0.34.dist-info/RECORD,,
+tgedr/dataops/validation/pyspark.py,sha256=4OEnA21_vSwB5HjaD5KZdfIjBnTn3KUAqvVGHnY-zNI,317
+tgedr_dataops-0.0.37.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
+tgedr_dataops-0.0.37.dist-info/METADATA,sha256=OGYBz2HJOKVIl5NlRqzjqjG4zHXmgFA2sErmeFCIAYk,639
+tgedr_dataops-0.0.37.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+tgedr_dataops-0.0.37.dist-info/top_level.txt,sha256=acugNvvENatFXbPxKQD9YI5PpzzehbAwyY1keiIkR7I,6
+tgedr_dataops-0.0.37.dist-info/RECORD,,