tgedr-dataops 0.0.32__tar.gz → 0.0.34__tar.gz

{tgedr_dataops-0.0.32/src/tgedr_dataops.egg-info → tgedr_dataops-0.0.34}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.32
+Version: 0.0.34
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas

{tgedr_dataops-0.0.32 → tgedr_dataops-0.0.34}/README.md

@@ -29,8 +29,9 @@ data operations related code
 - __LocalFsFileSource__: __source__ class used to retrieve local objects/files to another local fs location ([example](test/tgedr/dataops/source/test_localfs_file_source.py))
 - __S3FileSource__: __source__ class used to retrieve objects/files from s3 bucket to local fs location ([example](test/tgedr/dataops/source/test_s3_file_source.py))
 - __S3FileCopy__: __source__ class used to copy objects/files from an s3 bucket to another s3 bucket ([example](test/tgedr/dataops/source/test_s3_copy.py))
-- __S3DeltaTable__: __source__ class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe
-- __LocalDeltaTable__: __source__ class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe
+- __S3DeltaTable__: __source__ class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe ([example](test/tgedr/dataops/source/test_s3_delta_table.py))
+- __LocalDeltaTable__: __source__ class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe ([example](test/tgedr/dataops/source/test_local_delta_table.py))
+- __PdDfS3Source__: __source__ class used to read a pandas dataframe from, by default, a csv file in s3 ([example](test/tgedr/dataops/source/test_pd_df_s3_source.pypd))
 
 #### store
 - __Store__ : abstract class used to manage persistence, defining CRUD-like (CreateReadUpdateDelete) methods

{tgedr_dataops-0.0.32 → tgedr_dataops-0.0.34}/setup.py

@@ -4,7 +4,7 @@ import os
 from setuptools import setup, find_namespace_packages
 
 logger = logging.getLogger(__name__)
-VERSION = "0.0.32"
+VERSION = "0.0.34"
 logging.info(f"building version: {VERSION}")
 
 setup(

{tgedr_dataops-0.0.32 → tgedr_dataops-0.0.34}/src/tgedr/dataops/commons/utils_fs.py

@@ -52,6 +52,15 @@ def process_s3_path(path: str) -> Tuple[str, str]:
     return (bucket, key)
 
 
+def process_s3_url(url: str) -> Tuple[str, str, str]:
+    protocol = resolve_s3_protocol(url)
+    no_protocol_url = remove_s3_protocol(url)
+    path_elements = no_protocol_url.split("/")
+    bucket = path_elements[0]
+    key = "/".join(path_elements[1:])
+    return ("" if protocol is None else protocol, bucket, key)
+
+
 def hash_file(filepath, hash_func=hashlib.sha256) -> AnyStr:
     """Generate a hash for a file.
 

tgedr_dataops-0.0.34/src/tgedr/dataops/source/abstract_s3_file_source.py

@@ -0,0 +1,43 @@
+from abc import ABC
+import logging
+from typing import Any, Dict, List, Optional
+
+from tgedr.dataops.commons.s3_connector import S3Connector
+from tgedr.dataops.commons.utils_fs import process_s3_url
+from tgedr.dataops.source.source import Source, SourceException
+
+
+logger = logging.getLogger()
+
+
+class AbstractS3FileSource(Source, S3Connector, ABC):
+    """abstract class used to read file sources from s3"""
+
+    CONTEXT_KEY_URL = "url"
+    CONTEXT_KEY_SUFFIX = "suffix"
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        Source.__init__(self, config=config)
+        S3Connector.__init__(self)
+
+    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
+        logger.info(f"[list|in] ({context})")
+
+        result: List[str] = []
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
+
+        objs = self._client.list_objects_v2(Bucket=bucket, Prefix=key)
+        result = [
+            (protocol + bucket + "/" + entry["Key"]) for entry in objs["Contents"] if not (entry["Key"]).endswith("/")
+        ]
+
+        if self.CONTEXT_KEY_SUFFIX in context:
+            suffix: str = context[self.CONTEXT_KEY_SUFFIX]
+            result = [f for f in result if f.endswith(suffix)]
+
+        logger.debug(f"[list|out] => {result}")
+        logger.info(f"[list|out] => result len: {len(result)}")
+        return result

{tgedr_dataops-0.0.32 → tgedr_dataops-0.0.34}/src/tgedr/dataops/source/delta_table_source.py

@@ -3,8 +3,9 @@ import logging
 from typing import Any, Dict, List, Optional
 from pandas import DataFrame
 from deltalake import DeltaTable
+from deltalake.exceptions import TableNotFoundError
 
-from tgedr.dataops.source.source import Source, SourceException
+from tgedr.dataops.source.source import Source, SourceException, NoSourceException
 
 
 logger = logging.getLogger()
@@ -36,10 +37,13 @@ class DeltaTableSource(Source, ABC):
         if self.CONTEXT_KEY_COLUMNS in context:
             columns = context[self.CONTEXT_KEY_COLUMNS]
 
-        delta_table = DeltaTable(
-            table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
-        )
-        result = delta_table.to_pandas(columns=columns)
+        try:
+            delta_table = DeltaTable(
+                table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
+            )
+            result = delta_table.to_pandas(columns=columns)
+        except TableNotFoundError as tnfe:
+            raise NoSourceException(f"could not find delta table: {context[self.CONTEXT_KEY_URL]}")
 
         logger.info(f"[get|out] => {result}")
         return result

tgedr_dataops-0.0.34/src/tgedr/dataops/source/pd_df_s3_source.py

@@ -0,0 +1,51 @@
+from io import StringIO
+import logging
+from typing import Any, Dict, Optional
+import pandas as pd
+
+from tgedr.dataops.commons.utils_fs import process_s3_url
+from tgedr.dataops.source.abstract_s3_file_source import AbstractS3FileSource
+from tgedr.dataops.source.source import SourceException
+
+logger = logging.getLogger()
+
+
+class PdDfS3Source(AbstractS3FileSource):
+    """class used to read a pandas dataframe from a csv file in s3"""
+
+    CONTEXT_KEY_FILE_FORMAT = "file_format"
+    CONTEXT_KEY_SEPARATOR = "sep"
+    CONTEXT_KEY_NO_HEADER = "no_header"
+    CONTEXT_KEY_COLUMN_NAMES = "column_names"
+    CONTEXT_KEY_SCHEMA_TYPES = "schema_types"
+    DEFAULT_FORMAT = "csv"
+    DEFAULT_SEPARATOR = ","
+
+    def __init__(self, config: Optional[Dict[str, Any]] = None):
+        super().__init__(config=config)
+
+    def get(self, context: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
+        """retrieves a pandas dataframe, by default reading it from a csv,
+        you can ask for a different format using the context key 'file_format' (available formats: csv)"""
+        logger.info(f"[get|in] ({context})")
+        result: pd.DataFrame = None
+
+        if self.CONTEXT_KEY_URL not in context:
+            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
+
+        protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
+
+        obj = self._client.get_object(Bucket=bucket, Key=key)
+        data = obj["Body"].read().decode("utf-8")
+
+        header = 0 if self.CONTEXT_KEY_NO_HEADER not in context else None
+        names = None if self.CONTEXT_KEY_COLUMN_NAMES not in context else context[self.CONTEXT_KEY_COLUMN_NAMES]
+        dtype = None if self.CONTEXT_KEY_SCHEMA_TYPES not in context else context[self.CONTEXT_KEY_SCHEMA_TYPES]
+        sep = (
+            self.DEFAULT_SEPARATOR if self.CONTEXT_KEY_SEPARATOR not in context else context[self.CONTEXT_KEY_SEPARATOR]
+        )
+
+        result = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
+
+        logger.info(f"[get|out] => {result}")
+        return result

{tgedr_dataops-0.0.32 → tgedr_dataops-0.0.34}/src/tgedr/dataops/source/source.py

@@ -8,6 +8,10 @@ class SourceException(Exception):
     pass
 
 
+class NoSourceException(SourceException):
+    pass
+
+
 class SourceInterface(metaclass=abc.ABCMeta):
     """
     def get(self, context: Optional[Dict[str, Any]] = None) -> Any:

{tgedr_dataops-0.0.32 → tgedr_dataops-0.0.34/src/tgedr_dataops.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tgedr-dataops
-Version: 0.0.32
+Version: 0.0.34
 Summary: data operations related code
 Home-page: https://github.com/jtviegas-sandbox/dataops
 Author: joao tiago viegas

{tgedr_dataops-0.0.32 → tgedr_dataops-0.0.34}/src/tgedr_dataops.egg-info/SOURCES.txt

@@ -17,9 +17,11 @@ src/tgedr/dataops/sink/local_fs_file_sink.py
 src/tgedr/dataops/sink/s3_file_sink.py
 src/tgedr/dataops/sink/sink.py
 src/tgedr/dataops/source/__init__.py
+src/tgedr/dataops/source/abstract_s3_file_source.py
 src/tgedr/dataops/source/delta_table_source.py
 src/tgedr/dataops/source/local_delta_table.py
 src/tgedr/dataops/source/local_fs_file_source.py
+src/tgedr/dataops/source/pd_df_s3_source.py
 src/tgedr/dataops/source/s3_delta_table.py
 src/tgedr/dataops/source/s3_file_copy.py
 src/tgedr/dataops/source/s3_file_source.py