tgedr-dataops 0.0.34__tar.gz → 0.0.37__tar.gz

{tgedr_dataops-0.0.34/src/tgedr_dataops.egg-info → tgedr_dataops-0.0.37}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.34
+Version: 0.0.37
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas
@@ -17,3 +17,4 @@ Requires-Dist: s3fs==2024.5.0
 Requires-Dist: boto3==1.34.106
 Requires-Dist: great_expectations==0.18.10
 Requires-Dist: deltalake~=0.16.4
+Requires-Dist: openpyxl==3.1.2

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37}/README.md

@@ -28,10 +28,11 @@ data operations related code
 - __Source__: abstract **source** class defining methods (`list` and `get`) to manage retrieval of data from somewhere as defined by implementing classes
 - __LocalFsFileSource__: __source__ class used to retrieve local objects/files to another local fs location ([example](test/tgedr/dataops/source/test_localfs_file_source.py))
 - __S3FileSource__: __source__ class used to retrieve objects/files from s3 bucket to local fs location ([example](test/tgedr/dataops/source/test_s3_file_source.py))
+- __S3FileExtendedSource__: __source__ class used to retrieve objects/files from s3 bucket to local fs location with the extra method `get_metadata` providing sile metadata ("LastModified", "ContentLength", "ETag", "VersionId", "ContentType")([example](test/tgedr/dataops/source/test_s3_file_extended_source.py))
 - __S3FileCopy__: __source__ class used to copy objects/files from an s3 bucket to another s3 bucket ([example](test/tgedr/dataops/source/test_s3_copy.py))
 - __S3DeltaTable__: __source__ class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe ([example](test/tgedr/dataops/source/test_s3_delta_table.py))
 - __LocalDeltaTable__: __source__ class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe ([example](test/tgedr/dataops/source/test_local_delta_table.py))
-- __PdDfS3Source__: __source__ class used to read a pandas dataframe from, by default, a csv file in s3 ([example](test/tgedr/dataops/source/test_pd_df_s3_source.pypd))
+- __PdDfS3Source__: __source__ class used to read a pandas dataframe from s3, whether a csv or an excel (xslx) file ([example csv](test/tgedr/dataops/source/test_pd_df_s3_source_csv.py), [example excel](test/tgedr/dataops/source/test_pd_df_s3_source_excel.py))
 
 #### store
 - __Store__ : abstract class used to manage persistence, defining CRUD-like (CreateReadUpdateDelete) methods
@@ -41,9 +42,9 @@ data operations related code
 - __SparkDeltaStore__ : __store__ implementation for pyspark distributed processing with delta table format ([example](test/tgedr/dataops/store/test_spark_delta.py))
 
 #### validation
-- __DataValidation__ : abstract class defining a `validate` method to perform data validation, currently using Great Expectations library
-- __pandas.Impl__ : __DataValidation__ implementation to validate pandas dataframes with Great Expectations library ([example](test/tgedr/dataops/validation/test_validation.py))
-
+- __DataValidation__ : abstract class defining a `validate` method to perform data validation, currently using [Great Expectations library](https://greatexpectations.io/expectations/)
+- __pandas.Impl__ : __DataValidation__ implementation to validate pandas dataframes with Great Expectations library ([example](test/tgedr/dataops/validation/test_pandas.py))
+- __pyspark.Impl__ : __DataValidation__ implementation to validate pyspark dataframes with Great Expectations library ([example](test/tgedr/dataops/validation/test_pyspark.py))
 
 
 ## installation

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37}/setup.py

@@ -4,7 +4,7 @@ import os
 from setuptools import setup, find_namespace_packages
 
 logger = logging.getLogger(__name__)
-VERSION = "0.0.34"
+VERSION = "0.0.37"
 logging.info(f"building version: {VERSION}")
 
 setup(
@@ -29,7 +29,8 @@ setup(
         "s3fs==2024.5.0",
         "boto3==1.34.106",
         "great_expectations==0.18.10",
-        "deltalake~=0.16.4"
+        "deltalake~=0.16.4",
+        "openpyxl==3.1.2"
     ],
     python_requires='>=3.9',
 )

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37}/src/tgedr/dataops/source/pd_df_s3_source.py

@@ -19,6 +19,7 @@ class PdDfS3Source(AbstractS3FileSource):
     CONTEXT_KEY_COLUMN_NAMES = "column_names"
     CONTEXT_KEY_SCHEMA_TYPES = "schema_types"
     DEFAULT_FORMAT = "csv"
+    FORMATS = ["csv", "xlsx"]
     DEFAULT_SEPARATOR = ","
 
     def __init__(self, config: Optional[Dict[str, Any]] = None):
@@ -33,6 +34,23 @@ class PdDfS3Source(AbstractS3FileSource):
         if self.CONTEXT_KEY_URL not in context:
             raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
 
+        format: str = self.DEFAULT_FORMAT
+        if self.CONTEXT_KEY_FILE_FORMAT in context:
+            format = context[self.CONTEXT_KEY_FILE_FORMAT]
+            if format not in self.FORMATS:
+                raise SourceException(f"[get] invalid format: {format}")
+
+        if "csv" == format:
+            result = self.__read_csv(context=context)
+        else:
+            result = self.__read_excel(context=context)
+
+        logger.info(f"[get|out] => {result}")
+        return result
+
+    def __read_csv(self, context: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        logger.info(f"[__read_csv|in] ({context})")
+
         protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
 
         obj = self._client.get_object(Bucket=bucket, Key=key)
@@ -45,7 +63,14 @@ class PdDfS3Source(AbstractS3FileSource):
             self.DEFAULT_SEPARATOR if self.CONTEXT_KEY_SEPARATOR not in context else context[self.CONTEXT_KEY_SEPARATOR]
         )
 
-        result = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
+        result: pd.DataFrame = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
 
-        logger.info(f"[get|out] => {result}")
+        logger.info(f"[__read_csv|out] => {result}")
+        return result
+
+    def __read_excel(self, context: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        logger.info(f"[__read_excel|in] ({context})")
+        src = context[self.CONTEXT_KEY_URL]
+        result: pd.DataFrame = pd.read_excel(src, engine="openpyxl")
+        logger.info(f"[__read_excel|out] => {result}")
         return result

tgedr_dataops-0.0.37/src/tgedr/dataops/source/s3_file_extended_source.py

@@ -0,0 +1,39 @@
+import logging
+from typing import Any, Dict, Optional
+
+from tgedr.dataops.source.s3_file_source import S3FileSource
+from tgedr.dataops.source.source import SourceException
+from tgedr.dataops.commons.utils_fs import process_s3_path
+
+
+logger = logging.getLogger(__name__)
+
+
+class S3FileExtendedSource(S3FileSource):
+    """class used to retrieve objects/files from s3 bucket to local fs location"""
+
+    METADATA_KEYS = ["LastModified", "ContentLength", "ETag", "VersionId", "ContentType"]
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config=config)
+
+    def get_metadata(self, context: Optional[Dict[str, Any]] = None) -> Dict[str, Any]:
+        logger.info(f"[get_metadata|in] ({context})")
+
+        result: Dict[str, Any] = {}
+        if self.CONTEXT_KEY_SOURCE not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_SOURCE}")
+
+        bucket, key = process_s3_path(context[self.CONTEXT_KEY_SOURCE])
+
+        o = self._client.head_object(Bucket=bucket, Key=key)
+
+        for key in list(o.keys()):
+            if key in self.METADATA_KEYS:
+                if key == "LastModified":
+                    result[key] = int(o[key].timestamp())
+                else:
+                    result[key] = o[key]
+
+        logger.info(f"[get_metadata|out] => result len: {result}")
+        return result

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37}/src/tgedr/dataops/source/s3_file_source.py

@@ -22,21 +22,6 @@ class S3FileSource(Source, S3Connector):
         Source.__init__(self, config=config)
         S3Connector.__init__(self)
 
-    def __derive_local_file(self, target: str, isdir: bool, file: str):
-        logger.debug(f"[__derive_local_file|in] ({target}, {isdir}, {file})")
-
-        if isdir:
-            result = os.path.join(target, file)
-            basedir = os.path.dirname(result)
-            if not os.path.exists(basedir):
-                logger.debug(f"[__derive_local_file] creating folder: {basedir}")
-                os.mkdir(basedir)
-        else:
-            result = target
-
-        logger.debug(f"[__derive_local_file|out] => {result}")
-        return result
-
     def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
         logger.info(f"[list|in] ({context})")
 

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37}/src/tgedr/dataops/store/s3_single_partition_parquet.py

@@ -6,7 +6,7 @@ import pandas as pd
 import pyarrow as pa
 
 from tgedr.dataops.store.fs_single_partition_parquet import FsSinglePartitionParquetStore
-from src.nn.gs.ss.dataops.commons.utils_fs import remove_s3_protocol
+from tgedr.dataops.commons.utils_fs import remove_s3_protocol
 
 
 logger = logging.getLogger(__name__)

tgedr_dataops-0.0.37/src/tgedr/dataops/validation/pyspark.py

@@ -0,0 +1,10 @@
+from typing import Any
+from great_expectations.dataset.dataset import Dataset
+
+from tgedr.dataops.validation.abs import DataValidation
+from great_expectations.dataset.sparkdf_dataset import SparkDFDataset
+
+
+class Impl(DataValidation):
+    def _get_dataset(self, df: Any) -> Dataset:
+        return SparkDFDataset(df)

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37/src/tgedr_dataops.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.34
+Version: 0.0.37
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas
@@ -17,3 +17,4 @@ Requires-Dist: s3fs==2024.5.0
 Requires-Dist: boto3==1.34.106
 Requires-Dist: great_expectations==0.18.10
 Requires-Dist: deltalake~=0.16.4
+Requires-Dist: openpyxl==3.1.2

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37}/src/tgedr_dataops.egg-info/SOURCES.txt

@@ -24,6 +24,7 @@ src/tgedr/dataops/source/local_fs_file_source.py
 src/tgedr/dataops/source/pd_df_s3_source.py
 src/tgedr/dataops/source/s3_delta_table.py
 src/tgedr/dataops/source/s3_file_copy.py
+src/tgedr/dataops/source/s3_file_extended_source.py
 src/tgedr/dataops/source/s3_file_source.py
 src/tgedr/dataops/source/source.py
 src/tgedr/dataops/store/fs_single_partition_parquet.py
@@ -33,6 +34,7 @@ src/tgedr/dataops/store/spark_delta.py
 src/tgedr/dataops/store/store.py
 src/tgedr/dataops/validation/abs.py
 src/tgedr/dataops/validation/pandas.py
+src/tgedr/dataops/validation/pyspark.py
 src/tgedr_dataops.egg-info/PKG-INFO
 src/tgedr_dataops.egg-info/SOURCES.txt
 src/tgedr_dataops.egg-info/dependency_links.txt

{tgedr_dataops-0.0.34 → tgedr_dataops-0.0.37}/src/tgedr_dataops.egg-info/requires.txt

@@ -4,3 +4,4 @@ s3fs==2024.5.0
 boto3==1.34.106
 great_expectations==0.18.10
 deltalake~=0.16.4
+openpyxl==3.1.2