tgedr-dataops 0.0.36__py3-none-any.whl → 1.0.1__py3-none-any.whl

tgedr/dataops/source/delta_table_source.py

@@ -1,49 +0,0 @@
-from abc import ABC, abstractmethod
-import logging
-from typing import Any, Dict, List, Optional
-from pandas import DataFrame
-from deltalake import DeltaTable
-from deltalake.exceptions import TableNotFoundError
-
-from tgedr.dataops.source.source import Source, SourceException, NoSourceException
-
-
-logger = logging.getLogger()
-
-
-class DeltaTableSource(Source, ABC):
-    """abstract class used to read delta lake format datasets returning a pandas dataframe"""
-
-    CONTEXT_KEY_URL: str = "url"
-    CONTEXT_KEY_COLUMNS: str = "columns"
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        super().__init__(config=config)
-
-    @property
-    @abstractmethod
-    def _storage_options(self):
-        return None
-
-    def get(self, context: Optional[Dict[str, Any]] = None) -> DataFrame:
-        """retrieves a delta lake table"""
-        logger.info(f"[get|in] ({context})")
-        result: DataFrame = None
-
-        if self.CONTEXT_KEY_URL not in context:
-            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
-
-        columns: List[str] = None
-        if self.CONTEXT_KEY_COLUMNS in context:
-            columns = context[self.CONTEXT_KEY_COLUMNS]
-
-        try:
-            delta_table = DeltaTable(
-                table_uri=context[self.CONTEXT_KEY_URL], storage_options=self._storage_options, without_files=True
-            )
-            result = delta_table.to_pandas(columns=columns)
-        except TableNotFoundError as tnfe:
-            raise NoSourceException(f"could not find delta table: {context[self.CONTEXT_KEY_URL]}")
-
-        logger.info(f"[get|out] => {result}")
-        return result

tgedr/dataops/source/local_delta_table.py

@@ -1,47 +0,0 @@
-import logging
-import os
-import re
-from typing import Any, Dict, List, Optional
-import glob
-
-from tgedr.dataops.source.delta_table_source import DeltaTableSource
-from tgedr.dataops.source.source import SourceException
-
-
-logger = logging.getLogger()
-
-
-class LocalDeltaTable(DeltaTableSource):
-    """class used to read delta lake format datasets from local fs with python only, pyspark not needed, returning a pandas dataframe"""
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        super().__init__(config=config)
-
-    @property
-    def _storage_options(self):
-        return None
-
-    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
-        """lists the available delta lake datasets in the url provided"""
-        logger.info(f"[list|in] ({context})")
-
-        result: List[str] = []
-        if self.CONTEXT_KEY_URL not in context:
-            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
-
-        url = context[self.CONTEXT_KEY_URL]
-        if not os.path.isdir(url):
-            raise SourceException(f"not a delta lake url: {url}")
-
-        matches: set[str] = set()
-        pattern: str = f".*{url}/(.*)/_delta_log/.*"
-        for entry in glob.iglob(url + "**/**", recursive=True):
-            match = re.search(pattern, entry)
-            if match:
-                matches.add(match.group(1))
-
-        result = list(matches)
-
-        logger.info(f"[list] result: {result}")
-        logger.info(f"[list|out] => result len: {len(result)}")
-        return result

tgedr/dataops/source/local_fs_file_source.py

@@ -1,71 +0,0 @@
-import logging
-import os
-import shutil
-from typing import Any, Dict, List, Optional
-
-from tgedr.dataops.source.source import Source, SourceException
-
-
-logger = logging.getLogger(__name__)
-
-
-class LocalFsFileSource(Source):
-    """source class used to retrieve local objects/files to a another local fs location"""
-
-    CONTEXT_KEY_SOURCE = "source"
-    CONTEXT_KEY_TARGET = "target"
-    CONTEXT_KEY_SUFFIX = "suffix"
-    CONTEXT_KEY_FILES = "files"
-
-    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
-        logger.info(f"[list|in] ({context})")
-        result: List[str] = []
-        if self.CONTEXT_KEY_SOURCE not in context:
-            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_SOURCE}")
-
-        source = context[self.CONTEXT_KEY_SOURCE]
-        if os.path.isdir(source):
-            suffix = None
-            if self.CONTEXT_KEY_SUFFIX in context:
-                suffix = context[self.CONTEXT_KEY_SUFFIX]
-                result: List[str] = [os.path.join(source, file) for file in os.listdir(source) if file.endswith(suffix)]
-            else:
-                result: List[str] = [os.path.join(source, file) for file in os.listdir(source)]
-        elif os.path.isfile(source):
-            result: List[str] = [source]
-
-        logger.debug(f"[list|out] => {result}")
-        logger.info(f"[list|out] => result len: {len(result)}")
-        return result
-
-    def get(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        logger.info(f"[get|in] ({context})")
-
-        if self.CONTEXT_KEY_FILES not in context or self.CONTEXT_KEY_TARGET not in context:
-            raise SourceException(f"{self.CONTEXT_KEY_FILES} and {self.CONTEXT_KEY_TARGET} must be provided in config")
-        files = context[self.CONTEXT_KEY_FILES]
-        target = context[self.CONTEXT_KEY_TARGET]
-
-        if "list" != type(files).__name__:
-            if "string" == type(files).__name__:
-                files = [files]
-            else:
-                raise SourceException("files argument must be a list of strings or a string")
-
-        target_is_dir: bool = False
-        if os.path.isdir(target):
-            target_is_dir = True
-
-        result: List[str] = []
-
-        for file in files:
-            basename = os.path.basename(file)
-            if target_is_dir:
-                new_file = os.path.join(target, basename)
-            else:
-                new_file = target
-            shutil.copy(file, new_file)
-            result.append(new_file)
-
-        logger.info("[get|out] => {result}")
-        return result

tgedr/dataops/source/pd_df_s3_source.py

@@ -1,51 +0,0 @@
-from io import StringIO
-import logging
-from typing import Any, Dict, Optional
-import pandas as pd
-
-from tgedr.dataops.commons.utils_fs import process_s3_url
-from tgedr.dataops.source.abstract_s3_file_source import AbstractS3FileSource
-from tgedr.dataops.source.source import SourceException
-
-logger = logging.getLogger()
-
-
-class PdDfS3Source(AbstractS3FileSource):
-    """class used to read a pandas dataframe from a csv file in s3"""
-
-    CONTEXT_KEY_FILE_FORMAT = "file_format"
-    CONTEXT_KEY_SEPARATOR = "sep"
-    CONTEXT_KEY_NO_HEADER = "no_header"
-    CONTEXT_KEY_COLUMN_NAMES = "column_names"
-    CONTEXT_KEY_SCHEMA_TYPES = "schema_types"
-    DEFAULT_FORMAT = "csv"
-    DEFAULT_SEPARATOR = ","
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        super().__init__(config=config)
-
-    def get(self, context: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
-        """retrieves a pandas dataframe, by default reading it from a csv,
-        you can ask for a different format using the context key 'file_format' (available formats: csv)"""
-        logger.info(f"[get|in] ({context})")
-        result: pd.DataFrame = None
-
-        if self.CONTEXT_KEY_URL not in context:
-            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
-
-        protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
-
-        obj = self._client.get_object(Bucket=bucket, Key=key)
-        data = obj["Body"].read().decode("utf-8")
-
-        header = 0 if self.CONTEXT_KEY_NO_HEADER not in context else None
-        names = None if self.CONTEXT_KEY_COLUMN_NAMES not in context else context[self.CONTEXT_KEY_COLUMN_NAMES]
-        dtype = None if self.CONTEXT_KEY_SCHEMA_TYPES not in context else context[self.CONTEXT_KEY_SCHEMA_TYPES]
-        sep = (
-            self.DEFAULT_SEPARATOR if self.CONTEXT_KEY_SEPARATOR not in context else context[self.CONTEXT_KEY_SEPARATOR]
-        )
-
-        result = pd.read_csv(StringIO(data), sep=sep, header=header, names=names, dtype=dtype)
-
-        logger.info(f"[get|out] => {result}")
-        return result

tgedr/dataops/source/s3_delta_table.py

@@ -1,75 +0,0 @@
-import logging
-import re
-from typing import Any, Dict, List, Optional
-
-from tgedr.dataops.commons.s3_connector import S3Connector
-from tgedr.dataops.commons.utils_fs import remove_s3_protocol, resolve_s3_protocol
-from tgedr.dataops.source.delta_table_source import DeltaTableSource
-from tgedr.dataops.source.source import SourceException
-
-
-logger = logging.getLogger()
-
-
-class S3DeltaTable(DeltaTableSource, S3Connector):
-    """class used to read delta lake format datasets from s3 bucket with python only, pyspark not needed, returning a pandas dataframe"""
-
-    CONFIG_KEY_AWS_ACCESS_KEY_ID: str = "AWS_ACCESS_KEY_ID"
-    CONFIG_KEY_AWS_SECRET_ACCESS_KEY: str = "AWS_SECRET_ACCESS_KEY"
-    CONFIG_KEY_AWS_SESSION_TOKEN: str = "AWS_SESSION_TOKEN"
-    CONFIG_KEY_AWS_REGION: str = "AWS_REGION"
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        DeltaTableSource.__init__(self, config=config)
-        S3Connector.__init__(self)
-
-    @property
-    def _storage_options(self):
-        result = None
-        if (self._config is not None) and all(
-            element in list(self._config.keys())
-            for element in [
-                self.CONFIG_KEY_AWS_ACCESS_KEY_ID,
-                self.CONFIG_KEY_AWS_SECRET_ACCESS_KEY,
-                self.CONFIG_KEY_AWS_SESSION_TOKEN,
-                self.CONFIG_KEY_AWS_REGION,
-            ]
-        ):
-            result = {
-                "AWS_ACCESS_KEY_ID": self._config[self.CONFIG_KEY_AWS_ACCESS_KEY_ID],
-                "AWS_SECRET_ACCESS_KEY": self._config[self.CONFIG_KEY_AWS_SECRET_ACCESS_KEY],
-                "AWS_SESSION_TOKEN": self._config[self.CONFIG_KEY_AWS_SESSION_TOKEN],
-                "AWS_REGION": self._config[self.CONFIG_KEY_AWS_REGION],
-            }
-
-        return result
-
-    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
-        """lists the available delta lake datasets in the url provided"""
-        logger.info(f"[list|in] ({context})")
-
-        result: List[str] = []
-        if self.CONTEXT_KEY_URL not in context:
-            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
-
-        s3_protocol: str = resolve_s3_protocol(context[self.CONTEXT_KEY_URL])
-        protocol = "" if s3_protocol is None else s3_protocol
-
-        path = remove_s3_protocol(context[self.CONTEXT_KEY_URL])
-        path_elements = path.split("/")
-        bucket = path_elements[0]
-        key = "/".join(path_elements[1:])
-
-        matches: set[str] = set()
-        pattern: str = f".*{key}/(.*)/_delta_log/.*"
-        for entry in self._client.list_objects_v2(Bucket=bucket, Prefix=key)["Contents"]:
-            output_key: str = entry["Key"]
-            match = re.search(pattern, output_key)
-            if match:
-                matches.add(f"{key}/{match.group(1)}")
-
-        result = list(matches)
-
-        logger.info(f"[list] result: {result}")
-        logger.info(f"[list|out] => result len: {len(result)}")
-        return result

tgedr/dataops/source/source.py

@@ -1,51 +0,0 @@
-import abc
-from typing import Any, Dict, Optional
-
-from tgedr.dataops.chain import Chain
-
-
-class SourceException(Exception):
-    pass
-
-
-class NoSourceException(SourceException):
-    pass
-
-
-class SourceInterface(metaclass=abc.ABCMeta):
-    """
-    def get(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()
-    """
-
-    @classmethod
-    def __subclasshook__(cls, subclass):
-        return (
-            hasattr(subclass, "get")
-            and callable(subclass.get)
-            and hasattr(subclass, "list")
-            and callable(subclass.list)
-            or NotImplemented
-        )
-
-
-@SourceInterface.register
-class Source(abc.ABC):
-    """abstract class defining methods ('list' and 'get') to manage retrieval of data from somewhere as defined by implementing classes"""
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        self._config = config
-
-    @abc.abstractmethod
-    def get(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def list(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()
-
-
-@SourceInterface.register
-class SourceChain(Chain, abc.ABC):
-    def execute(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        return self.get(context=context)

tgedr/dataops/store/fs_single_partition_parquet.py

@@ -1,231 +0,0 @@
-from abc import ABC, abstractmethod
-import logging
-import os
-from typing import Any, Dict, List, Optional
-import pandas as pd
-import pyarrow as pa
-import pyarrow.parquet as pq
-import pyarrow.compute as pc
-
-from tgedr.dataops.store.store import Store, StoreException
-
-
-logger = logging.getLogger(__name__)
-
-
-def pandas_mapper(arrow_type):
-    if pa.types.is_int64(arrow_type):
-        return pd.Int64Dtype()
-    if pa.types.is_float64(arrow_type):
-        return pd.Float64Dtype()
-    if pa.types.is_string(arrow_type):
-        return pd.StringDtype()
-    # suggest default behavior
-    return None
-
-
-class FsSinglePartitionParquetStore(Store, ABC):
-    """abstract store implementation defining persistence on parquet files with an optional single partition,
-    regardless of the location it should persist"""
-
-    @property
-    @abstractmethod
-    def fs(self):
-        """abstract method providing a filesystem implementation (local, s3, etc...)"""
-        raise NotImplementedError()
-
-    @abstractmethod
-    def _rmdir(self, key):
-        raise NotImplementedError()
-
-    @abstractmethod
-    def _exists(self, key) -> bool:
-        raise NotImplementedError()
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        Store.__init__(self, config)
-        self._fs = None
-
-    def get(
-        self,
-        key: str,
-        filter: callable = None,
-        filters: List[tuple[str, str, List[str]]] = None,
-        schema: pa.Schema = None,
-    ) -> pd.DataFrame:
-        """
-        reads a pandas dataframe from somewhere (key), depending on implementation, eventually enforcing a schema and
-        allowing filtering of data
-
-        Parameters:
-        key (str): location/url/path where data should be persisted
-        filter (Array or array-like or Expression): filter expression (see: https://arrow.apache.org/docs/python/generated/pyarrow.Table.html#pyarrow.Table.filter)
-        filters (pyarrow.compute.Expression or List[Tuple] or List[List[Tuple]]): filter expression (see: https://arrow.apache.org/docs/python/generated/pyarrow.parquet.read_table.html)
-        schema: data schema to enforce while reading (see: https://arrow.apache.org/docs/python/generated/pyarrow.Schema.html#pyarrow.Schema)
-
-        Returns:
-        pandas.DataFrame: the dataframe
-        """
-        schema_msg_segment = "0" if schema is None else str(len(schema))
-        logger.info(f"[get|in] ({key}, {filter}, {filters}, schema len:{schema_msg_segment})")
-        logger.debug(f"[get|in] ({key}, {filter}, {filters}, {schema})")
-        table = pq.read_table(key, filesystem=self.fs, filters=filters, schema=schema)
-        if filter is not None:
-            table = table.filter(filter)
-        result = table.to_pandas(types_mapper=pandas_mapper)
-        logger.info(f"[get|out] => {result.shape}")
-        return result
-
-    def delete(
-        self,
-        key: str,
-        partition_field: Optional[str] = None,
-        partition_values: Optional[List[str]] = None,
-        kv_dict: Optional[Dict[str, List[Any]]] = None,
-        schema: pa.Schema = None,
-    ):
-        """
-        removes partitions, full or partial, or deletes partial values or a full dataset
-        from a parquet storage somewhere (key), depending on implementation
-
-        Parameters:
-        key (str): location/url/path where data is persisted
-        partition_field (str): name of the partition field in the dataset
-        partition_values (str): partition values to delete
-        kv_dict (Dict[str, List[Any]]): key-value map defining the fields and array of values to become the deletion filter
-        schema: data schema to enforce if reading is required (see: https://arrow.apache.org/docs/python/generated/pyarrow.Schema.html#pyarrow.Schema)
-        """
-        schema_msg_segment = "0" if schema is None else str(len(schema))
-        logger.info(
-            f"[delete|in] ({key}, {partition_field}, {partition_values}, {kv_dict}, schema len:{schema_msg_segment})"
-        )
-        logger.debug(f"[delete|in] ({key}, {partition_field}, {partition_values}, {kv_dict}, {schema})")
-
-        if partition_values is not None and partition_field is not None:
-            self._remove_partitions(key, partition_field=partition_field, partition_values=partition_values)
-        elif kv_dict is not None and partition_field is not None:
-            table = pq.read_table(key, filesystem=self.fs, schema=schema)
-            for k, v in kv_dict.items():
-                filter_condition = ~pc.is_in(pc.field(k), pa.array(v))
-                table = table.filter(filter_condition)
-            self.delete(key, schema=schema)
-            pq.write_to_dataset(
-                table,
-                root_path=key,
-                partition_cols=[partition_field],
-                existing_data_behavior="delete_matching",
-                filesystem=self.fs,
-                schema=schema,
-            )
-        else:
-            self._rmdir(key)
-
-        logger.info("[delete|out]")
-
-    def save(
-        self,
-        df: pd.DataFrame,
-        key: str,
-        partition_field: Optional[str] = None,
-        append: bool = False,
-        replace_partitions: bool = False,
-        schema: Any = None,
-    ):
-        """
-        saves a pandas dataframe in parquet format somewhere (key), depending on implementation
-
-        Parameters:
-        df (pandas.DataFrame): the dataframe to be saved
-        key (str): location/url/path where data is persisted
-        partition_field (str): name of the partition field in the dataset
-        append (bool): if data should be appended, otherwise will overwrite
-        replace_partitions (bool): if partitions should be replaced, this will delete the data existent on those partitions completely
-        schema: data schema to enforce if reading is required (see: https://arrow.apache.org/docs/python/generated/pyarrow.Schema.html#pyarrow.Schema)
-        """
-        schema_msg_segment = "0" if schema is None else str(len(schema))
-        logger.info(
-            f"[save|in] ({df.shape}, {key}, {partition_field}, {append}, {replace_partitions}, schema len:{schema_msg_segment})"
-        )
-        logger.debug(f"[save|in] ({df}, {key}, {partition_field}, {append}, {replace_partitions}, {schema})")
-
-        if schema is not None and isinstance(schema, pa.lib.Schema):
-            # we will order the columns based on the schema
-            columns = [col for col in schema.names]
-            df = df[columns]
-
-        if replace_partitions and append:
-            raise StoreException(f"cannot request for replace_partitions and append at the same time")
-
-        if append:
-            pq.write_to_dataset(
-                pa.Table.from_pandas(df, preserve_index=False),
-                root_path=key,
-                partition_cols=[partition_field],
-                filesystem=self.fs,
-                schema=schema,
-            )
-        elif replace_partitions:
-            partitions = df[partition_field].unique().tolist()
-            self._remove_partitions(key, partition_field, partitions)
-            pq.write_to_dataset(
-                pa.Table.from_pandas(df, preserve_index=False),
-                root_path=key,
-                partition_cols=[partition_field],
-                existing_data_behavior="delete_matching",
-                filesystem=self.fs,
-                schema=schema,
-            )
-        else:
-            self.delete(key)
-            pq.write_to_dataset(
-                pa.Table.from_pandas(df, preserve_index=False),
-                root_path=key,
-                partition_cols=[partition_field],
-                existing_data_behavior="delete_matching",
-                filesystem=self.fs,
-                schema=schema,
-            )
-        logger.info("[save|out]")
-
-    def _remove_partitions(self, key: str, partition_field: str, partition_values: List[str]):
-        logger.debug(f"[_remove_partitions|in] ({key}, {partition_field}, {partition_values})")
-
-        for partition_value in partition_values:
-            partition_key = f"{partition_field}={partition_value}"
-            partition_path = os.path.join(key, partition_key)
-            self._rmdir(partition_path)
-
-        logger.debug("[_remove_partitions|out]")
-
-    def update(
-        self,
-        df: pd.DataFrame,
-        key: str,
-        key_fields: List[str],
-        partition_field: Optional[str] = None,
-        schema: Any = None,
-    ):
-        """
-        updates a pandas dataframe in parquet format somewhere (key), depending on implementation
-
-        Parameters:
-        df (pandas.DataFrame): the dataframe to be saved
-        key (str): location/url/path where data is persisted
-        key_fields (List[str]): primary fields of the dataset used to match the rows to update with the new dataset
-        partition_field (str): name of the partition field to enforce while saving
-        schema: data schema to enforce while reading and saving (see: https://arrow.apache.org/docs/python/generated/pyarrow.Schema.html#pyarrow.Schema)
-        """
-        schema_msg_segment = "0" if schema is None else str(len(schema))
-        logger.info(
-            f"[update|in] ({df.shape}, {key}, {key_fields}, {partition_field}, schema len:{schema_msg_segment})"
-        )
-        logger.debug(f"[update|in] ({df}, {key}, {key_fields}, {partition_field}, {schema})")
-
-        df0 = self.get(key, schema=schema)
-        match = pd.merge(df0.reset_index(), df.reset_index(), on=key_fields)
-        index_left = match["index_x"]
-        index_right = match["index_y"]
-        df0.iloc[index_left] = df.iloc[index_right]
-        self.save(df0, key, partition_field=partition_field, schema=schema)
-
-        logger.info(f"[update|out]")

tgedr/dataops/store/local_fs_single_partition_parquet.py

@@ -1,24 +0,0 @@
-import logging
-from pyarrow import fs
-
-from tgedr.dataops.store.fs_single_partition_parquet import FsSinglePartitionParquetStore
-
-
-logger = logging.getLogger(__name__)
-
-
-class LocalFsSinglePartitionParquetStore(FsSinglePartitionParquetStore):
-    """FsSinglePartitionParquetStore implementation using local file system"""
-
-    @property
-    def fs(self):
-        if self._fs is None:
-            self._fs = fs.LocalFileSystem()
-        return self._fs
-
-    def _rmdir(self, key):
-        if self.fs.get_file_info(key).type.name == "Directory":
-            self.fs.delete_dir(key)
-
-    def _exists(self, key) -> bool:
-        return self.fs.get_file_info(key).type.name != "NotFound"