tgedr-dataops 0.0.32__py3-none-any.whl → 0.0.34__py3-none-any.whl

tgedr/dataops/commons/utils_fs.py

@@ -52,6 +52,15 @@ def process_s3_path(path: str) -> Tuple[str, str]:
     return (bucket, key)
 
 
+def process_s3_url(url: str) -> Tuple[str, str, str]:
+    protocol = resolve_s3_protocol(url)
+    no_protocol_url = remove_s3_protocol(url)
+    path_elements = no_protocol_url.split("/")
+    bucket = path_elements[0]
+    key = "/".join(path_elements[1:])
+    return ("" if protocol is None else protocol, bucket, key)
+
+
 def hash_file(filepath, hash_func=hashlib.sha256) -> AnyStr:
     """Generate a hash for a file.
 

tgedr/dataops/source/abstract_s3_file_source.py

@@ -0,0 +1,43 @@
+from abc import ABC
+import logging
+from typing import Any, Dict, List, Optional
+
+from tgedr.dataops.commons.s3_connector import S3Connector
+from tgedr.dataops.commons.utils_fs import process_s3_url
+from tgedr.dataops.source.source import Source, SourceException
+
+
+logger = logging.getLogger()
+
+
+class AbstractS3FileSource(Source, S3Connector, ABC):
+    """abstract class used to read file sources from s3"""
+
+    CONTEXT_KEY_URL = "url"
+    CONTEXT_KEY_SUFFIX = "suffix"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        Source.__init__(self, config=config)
+        S3Connector.__init__(self)
+
+    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
+        logger.info(f"[list|in] ({context})")
+
+        result: List[str] = []
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
+
+        objs = self._client.list_objects_v2(Bucket=bucket, Prefix=key)
+        result = [
+            (protocol + bucket + "/" + entry["Key"]) for entry in objs["Contents"] if not (entry["Key"]).endswith("/")
+        ]
+
+        if self.CONTEXT_KEY_SUFFIX in context:
+            suffix: str = context[self.CONTEXT_KEY_SUFFIX]
+            result = [f for f in result if f.endswith(suffix)]
+
+        logger.debug(f"[list|out] => {result}")
+        logger.info(f"[list|out] => result len: {len(result)}")
+        return result

tgedr/dataops/source/delta_table_source.py

@@ -3,8 +3,9 @@ import logging
 from typing import Any, Dict, List, Optional
 from pandas import DataFrame
 from deltalake import DeltaTable
+from deltalake.exceptions import TableNotFoundError
 
-from tgedr.dataops.source.source import Source, SourceException
+from tgedr.dataops.source.source import Source, SourceException, NoSourceException
 
 
 logger = logging.getLogger()
@@ -36,10 +37,13 @@ class DeltaTableSource(Source, ABC):
         if self.CONTEXT_KEY_COLUMNS in context:
             columns = context[self.CONTEXT_KEY_COLUMNS]
 
-        delta_table = DeltaTable(
-            table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
-        )
-        result = delta_table.to_pandas(columns=columns)
+        try:
+            delta_table = DeltaTable(
+                table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
+            )
+            result = delta_table.to_pandas(columns=columns)
+        except TableNotFoundError as tnfe:
+            raise NoSourceException(f"could not find delta table: {context[self.CONTEXT_KEY_URL]}")
 
         logger.info(f"[get|out] => {result}")
         return result

tgedr/dataops/source/pd_df_s3_source.py

@@ -0,0 +1,51 @@
+from io import StringIO
+import logging
+from typing import Any, Dict, Optional
+import pandas as pd
+
+from tgedr.dataops.commons.utils_fs import process_s3_url
+from tgedr.dataops.source.abstract_s3_file_source import AbstractS3FileSource
+from tgedr.dataops.source.source import SourceException
+
+logger = logging.getLogger()
+
+
+class PdDfS3Source(AbstractS3FileSource):
+    """class used to read a pandas dataframe from a csv file in s3"""
+
+    CONTEXT_KEY_FILE_FORMAT = "file_format"
+    CONTEXT_KEY_SEPARATOR = "sep"
+    CONTEXT_KEY_NO_HEADER = "no_header"
+    CONTEXT_KEY_COLUMN_NAMES = "column_names"
+    CONTEXT_KEY_SCHEMA_TYPES = "schema_types"
+    DEFAULT_FORMAT = "csv"
+    DEFAULT_SEPARATOR = ","
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config=config)
+
+    def get(self, context: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        """retrieves a pandas dataframe, by default reading it from a csv,
+        you can ask for a different format using the context key 'file_format' (available formats: csv)"""
+        logger.info(f"[get|in] ({context})")
+        result: pd.DataFrame = None
+
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
+
+        obj = self._client.get_object(Bucket=bucket, Key=key)
+        data = obj["Body"].read().decode("utf-8")
+
+        header = 0 if self.CONTEXT_KEY_NO_HEADER not in context else None
+        names = None if self.CONTEXT_KEY_COLUMN_NAMES not in context else context[self.CONTEXT_KEY_COLUMN_NAMES]
+        dtype = None if self.CONTEXT_KEY_SCHEMA_TYPES not in context else context[self.CONTEXT_KEY_SCHEMA_TYPES]
+        sep = (
+            self.DEFAULT_SEPARATOR if self.CONTEXT_KEY_SEPARATOR not in context else context[self.CONTEXT_KEY_SEPARATOR]
+        )
+
+        result = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
+
+        logger.info(f"[get|out] => {result}")
+        return result

tgedr/dataops/source/source.py

@@ -8,6 +8,10 @@ class SourceException(Exception):
     pass
 
 
+class NoSourceException(SourceException):
+    pass
+
+
 class SourceInterface(metaclass=abc.ABCMeta):
     """
     def get(self, context: Optional[Dict[str, Any]] = None) -> Any:

{tgedr_dataops-0.0.32.dist-info → tgedr_dataops-0.0.34.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.32
+Version: 0.0.34
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas

{tgedr_dataops-0.0.32.dist-info → tgedr_dataops-0.0.34.dist-info}/RECORD

@@ -6,20 +6,22 @@ tgedr/dataops/utils_reflection.py,sha256=W3eqrypMwqRsvevEZ9d6Zxw_GJL4d938lXiU1gI
 tgedr/dataops/commons/dataset.py,sha256=VIt385Zps78y6cbe54G4PLNuUA647S94UqokubM1gKI,637
 tgedr/dataops/commons/metadata.py,sha256=5uIurGkkREJuAlONav8_TyLVNY097Zix6LIdRxVE_uQ,5796
 tgedr/dataops/commons/s3_connector.py,sha256=AmD8l86ORbABamkquD8BMtMpgSfThkVbl9Kf54wR_pA,1194
-tgedr/dataops/commons/utils_fs.py,sha256=BS28O4GoU1FsqPM_S22FmDnXDHCpu3KPAwBxXTWwQxU,2042
+tgedr/dataops/commons/utils_fs.py,sha256=8gXPcdQx520pr1h7tn7m_Z1AojgWIdebNLCyiQgEZEU,2362
 tgedr/dataops/commons/utils_spark.py,sha256=VSFUcQt-yE54VHJkonveFwhM3ADTK3kfC3QfA34QOVI,3191
 tgedr/dataops/sink/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 tgedr/dataops/sink/local_fs_file_sink.py,sha256=JjFWwW_ufWF8I-PVPP1TleKmRj8Des-6A0xLqtbPYW8,1508
 tgedr/dataops/sink/s3_file_sink.py,sha256=m1D6SwuqYEVqvdA9XrL6nVlX_oWLRDT6v74mh_diuQM,2362
 tgedr/dataops/sink/sink.py,sha256=8rG3ZNpzeZ82Ac1IoPzkdQTs006IbG-k39APFCeXogk,1271
 tgedr/dataops/source/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-tgedr/dataops/source/delta_table_source.py,sha256=ElZezb6JeYRdkvY8Ryk1EJUvbUHCWRrlblmdUZZBSW4,1410
+tgedr/dataops/source/abstract_s3_file_source.py,sha256=AMnTWxs57Jni46jz5PvrZTRjd-Ah_TUZEP4Rcx7YziM,1513
+tgedr/dataops/source/delta_table_source.py,sha256=tXeJCM-QhssztEpyKGzb6zcM1AlaUwV1Y5d2RHCiOTU,1653
 tgedr/dataops/source/local_delta_table.py,sha256=Z9413skOSsNGfSNeBs-ETVXQJcW4bhS3MYmCwySo9Tc,1545
 tgedr/dataops/source/local_fs_file_source.py,sha256=wihK_wVV6xb8AlkCmZHSRlRmqqm6w9Aq-PI0oOnhn_k,2570
+tgedr/dataops/source/pd_df_s3_source.py,sha256=3Mc0VDGTOmk1HT951l6Qd_bqL0DpPjlfZk0JQjWh3JE,2092
 tgedr/dataops/source/s3_delta_table.py,sha256=d_FoFDNogjZifRQv8V2OmTXbPYymx9QaQcW15uX34pI,2974
 tgedr/dataops/source/s3_file_copy.py,sha256=DbHvstAqi23cywoG6nHpQxvzerrnepApOsv6zsELYNQ,3930
 tgedr/dataops/source/s3_file_source.py,sha256=C8Y0h89p1eBWNGxV4oTNzVLLwZoAoyshHuMlDKKqDA0,4072
-tgedr/dataops/source/source.py,sha256=xm9XNGG-WTo_EGdJ3cfNfSC-nuHeEO2XeKiPIhv4Qoc,1297
+tgedr/dataops/source/source.py,sha256=REeqluMGLMjoDWtdZthzUYkmVeHemSV7t9wjc6eTpJE,1350
 tgedr/dataops/store/fs_single_partition_parquet.py,sha256=CR3406emhxn33jjObnMotXEmZGfh4Iu5Ygv30FvkY6Y,9695
 tgedr/dataops/store/local_fs_single_partition_parquet.py,sha256=N_I96fqxQAp2fWBngoDci3aR1-kcmkWjIVRD0nUi07U,683
 tgedr/dataops/store/s3_single_partition_parquet.py,sha256=JwMhRO9403OLhepeAdbSDNhmM7I4dLoriQfr2IxymHE,3256
@@ -27,8 +29,8 @@ tgedr/dataops/store/spark_delta.py,sha256=AHqIKDi9axOKpMJHt4AiBGX8V2mFV_7vNPRpaw
 tgedr/dataops/store/store.py,sha256=uAuR7MWVdKRaisQ69rFqliLnJrsgpTzbsOh7uPmLSlI,1315
 tgedr/dataops/validation/abs.py,sha256=84HGUuh6k_uG-ON0bauR4lDBTfUeI3GmxOiWsMkTu3E,1521
 tgedr/dataops/validation/pandas.py,sha256=Vfr38f3txbTy098ufPUcRsCgrYu47Rg35upZ9IXcLSk,315
-tgedr_dataops-0.0.32.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
-tgedr_dataops-0.0.32.dist-info/METADATA,sha256=HlFWY3yp8v_ZMrDqXd1zQKtMbqYrDmQAK0VkA_UcMog,607
-tgedr_dataops-0.0.32.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
-tgedr_dataops-0.0.32.dist-info/top_level.txt,sha256=acugNvvENatFXbPxKQD9YI5PpzzehbAwyY1keiIkR7I,6
-tgedr_dataops-0.0.32.dist-info/RECORD,,
+tgedr_dataops-0.0.34.dist-info/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
+tgedr_dataops-0.0.34.dist-info/METADATA,sha256=ryzlSTJx7GtIQNpX3PL2QiZ0h-bsR_smUL-l_rdfXdw,607
+tgedr_dataops-0.0.34.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92
+tgedr_dataops-0.0.34.dist-info/top_level.txt,sha256=acugNvvENatFXbPxKQD9YI5PpzzehbAwyY1keiIkR7I,6
+tgedr_dataops-0.0.34.dist-info/RECORD,,