tgedr-dataops 0.0.36__py3-none-any.whl → 1.0.1__py3-none-any.whl

tgedr/dataops/chain.py

@@ -1,51 +0,0 @@
-import abc
-from typing import Any, Dict, Optional
-
-from tgedr.dataops.processor import Processor
-
-
-class ChainException(Exception):
-    pass
-
-
-class ChainInterface(metaclass=abc.ABCMeta):
-    @classmethod
-    def __subclasshook__(cls, subclass):
-        return (
-            hasattr(subclass, "next")
-            and callable(subclass.next)
-            and hasattr(subclass, "execute")
-            and callable(subclass.execute)
-        ) or NotImplemented
-
-
-class ChainMixin(abc.ABC):
-    def next(self, handler: "ChainMixin") -> "ChainMixin":
-        if "_next" not in self.__dict__ or self._next is None:
-            self._next: "ChainMixin" = handler
-        else:
-            self._next.next(handler)
-        return self
-
-    @abc.abstractmethod
-    def execute(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()
-
-
-class ProcessorChainMixin(ChainMixin):
-    def execute(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        self.process(context=context)
-        if "_next" in self.__dict__ and self._next is not None:
-            self._next.execute(context=context)
-
-
-@ChainInterface.register
-class ProcessorChain(ProcessorChainMixin, Processor):
-    pass
-
-
-@ChainInterface.register
-class Chain(ChainMixin, abc.ABC):
-    @abc.abstractmethod
-    def execute(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()

tgedr/dataops/commons/dataset.py

@@ -1,23 +0,0 @@
-from dataclasses import dataclass
-import json
-from pyspark.sql import DataFrame
-from tgedr.dataops.commons.metadata import Metadata
-
-
-@dataclass(frozen=True)
-class Dataset:
-    """
-    utility immutable class to wrap up a dataframe along with metadata
-    """
-
-    __slots__ = ["metadata", "data"]
-    metadata: Metadata
-    data: DataFrame
-
-    def as_dict(self) -> dict:
-        """serialize the dataset as a dictionary"""
-        return {"metadata": self.metadata.as_dict(), "data": str(self.data.__repr__)}
-
-    def __str__(self) -> str:
-        """serialize the dataset as a json string"""
-        return json.dumps(self.as_dict())

tgedr/dataops/commons/metadata.py

@@ -1,172 +0,0 @@
-from dataclasses import dataclass
-import json
-from typing import Dict, List, Optional, Union
-
-
-@dataclass(frozen=True)
-class FieldFrame:
-    """
-    class depicting a field values range, to be used in metadata
-
-    Parameters:
-        field (str): the name of the field
-        lower (Union[int, str, float]): field lower bound
-        upper (Union[int, str, float]): field upper bound
-    """
-
-    __slots__ = ["field", "lower", "upper"]
-    field: str
-    lower: Union[int, str, float]
-    upper: Union[int, str, float]
-
-    def as_dict(self) -> Dict[str, Union[int, str, float]]:
-        return {"field": self.field, "lower": self.lower, "upper": self.upper}
-
-    @staticmethod
-    def from_str(src: str) -> "FieldFrame":
-        r = json.loads(src)
-        field = r["field"]
-        lower = r["lower"]
-        upper = r["upper"]
-        return FieldFrame(field=field, lower=lower, upper=upper)
-
-    def __str__(self) -> str:
-        return json.dumps(self.as_dict())
-
-    def __eq__(self, other):
-        return self.field == other.field and self.lower == other.lower and self.upper == other.upper
-
-    def __gt__(self, other):
-        return self.field > other.field or (
-            self.field == other.field
-            and (self.lower > other.lower or (self.lower == other.lower and self.upper > other.upper))
-        )
-
-    def __ne__(self, other):
-        return not other == self
-
-    def __ge__(self, other):
-        return other == self or self > other
-
-    def __le__(self, other):
-        return other == self or self < other
-
-    def __lt__(self, other):
-        return other > self
-
-
-@dataclass(frozen=True)
-class Metadata:
-    """immutable class depicting dataset metadata
-
-    Parameters:
-        name (str): the name of the dataset
-        version (Optional[str]): version of this dataset, if available
-        framing (Optional[List[FieldFrame]]): multiple field frames
-        sources (Optional[List["Metadata"]]): metadatas related to the datasets sourcing this one
-    """
-
-    __slots__ = ["name", "version", "framing", "sources"]
-    name: str
-    version: Optional[str]
-    framing: Optional[List[FieldFrame]]
-    sources: Optional[List["Metadata"]]
-
-    def as_dict(self) -> dict:
-        result = {"name": self.name}
-        if self.version is not None:
-            result["version"] = self.version
-        if self.framing is not None:
-            result["framing"] = []
-            for f in self.framing:
-                (result["framing"]).append(f.as_dict())
-        if self.sources is not None:
-            result["sources"] = []
-            for source in self.sources:
-                (result["sources"]).append(source.as_dict())
-
-        return result
-
-    def __str__(self):
-        return json.dumps(self.as_dict())
-
-    def __eq__(self, other: object) -> bool:
-        return (
-            self.name == other.name
-            and (
-                (self.version is None and other.version is None)
-                or ((self.version is not None and other.version is not None) and self.version == other.version)
-            )
-            and (
-                (self.framing is None and other.framing is None)
-                or (
-                    (self.framing is not None and other.framing is not None)
-                    and sorted(self.framing) == sorted(other.framing)
-                )
-            )
-            and (
-                (self.sources is None and other.sources is None)
-                or (
-                    (self.sources is not None and other.sources is not None)
-                    and sorted(self.sources) == sorted(other.sources)
-                )
-            )
-        )
-
-    def __gt__(self, other):
-        return self.name > other.name or (
-            self.name == other.name
-            and (
-                ((self.version is not None and other.version is not None) and (self.version > other.version))
-                or (self.version is not None and other.version is None)
-                or (
-                    (
-                        (self.framing is not None and other.framing is not None)
-                        and (sorted(self.framing) > sorted(other.framing))
-                    )
-                    or (self.framing is not None and other.framing is None)
-                    or (
-                        (
-                            (self.sources is not None and other.sources is not None)
-                            and (sorted(self.sources) > sorted(other.sources))
-                        )
-                        or (self.sources is not None and other.sources is None)
-                    )
-                )
-            )
-        )
-
-    def __ne__(self, other):
-        return not other == self
-
-    def __ge__(self, other):
-        return other == self or self > other
-
-    def __le__(self, other):
-        return other == self or self < other
-
-    def __lt__(self, other):
-        return other > self
-
-    @staticmethod
-    def from_str(src: str) -> "Metadata":
-        r = json.loads(src)
-        name = r["name"]
-        version = None if "version" not in r else r["version"]
-
-        framing = None
-        framing_entries = None if "framing" not in r else r["framing"]
-        if framing_entries is not None:
-            framing = []
-            for framing_entry in framing_entries:
-                framing.append(FieldFrame.from_str(json.dumps(framing_entry)))
-
-        sources = None
-        sources_entries = None if "sources" not in r else r["sources"]
-        if sources_entries is not None:
-            sources = []
-            for source_entry in sources_entries:
-                source_entry_as_str = json.dumps(source_entry)
-                sources.append(Metadata.from_str(source_entry_as_str))
-
-        return Metadata(name=name, version=version, framing=framing, sources=sources)

tgedr/dataops/commons/utils_fs.py

@@ -1,85 +0,0 @@
-import tempfile
-import re
-from typing import Tuple, Union, AnyStr
-import hashlib
-
-
-def temp_dir(root: str = None, suffix: str = None, prefix: str = None) -> str:
-    return tempfile.mkdtemp(suffix=suffix, prefix=prefix, dir=root)
-
-
-def temp_file(
-    root: str = None, suffix: str = None, prefix: str = None, discard_handle: bool = True
-) -> Union[str, Tuple[int, str]]:
-    h, f = tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=root)
-    if discard_handle:
-        return f
-    else:
-        return (h, f)
-
-
-def resolve_url_protocol(url: str) -> str:
-    result = None
-    group_match = re.search("(.*://).*", url)
-    if group_match is not None:
-        result = group_match.group(1)
-    return result
-
-
-def resolve_s3_protocol(url: str) -> str:
-    result = None
-    group_match = re.search("(s3[a]?://).*", url)
-    if group_match is not None:
-        result = group_match.group(1)
-    return result
-
-
-def remove_s3_protocol(url: str) -> str:
-    if url.startswith("s3://"):
-        result = url[5:]
-    elif url.startswith("s3a://"):
-        result = url[6:]
-    else:
-        result = url
-    return result
-
-
-def process_s3_path(path: str) -> Tuple[str, str]:
-    no_protocol_path = remove_s3_protocol(path)
-    path_elements = no_protocol_path.split("/")
-    bucket = path_elements[0]
-    key = "/".join(path_elements[1:])
-    return (bucket, key)
-
-
-def process_s3_url(url: str) -> Tuple[str, str, str]:
-    protocol = resolve_s3_protocol(url)
-    no_protocol_url = remove_s3_protocol(url)
-    path_elements = no_protocol_url.split("/")
-    bucket = path_elements[0]
-    key = "/".join(path_elements[1:])
-    return ("" if protocol is None else protocol, bucket, key)
-
-
-def hash_file(filepath, hash_func=hashlib.sha256) -> AnyStr:
-    """Generate a hash for a file.
-
-    Args:
-        filepath (str): The path to the file.
-        hash_func: A hashlib hash function, e.g., hashlib.md5().
-
-    Returns:
-        str: The hexadecimal hash string of the file.
-    """
-    # Initialize the hash object
-    hasher = hash_func()
-
-    # Open the file in binary read mode
-    with open(filepath, "rb") as file:
-        # Read the file in chunks to avoid using too much memory
-        chunk_size = 8192
-        while chunk := file.read(chunk_size):
-            hasher.update(chunk)
-
-    # Return the hexadecimal digest of the hash
-    return hasher.hexdigest()

tgedr/dataops/commons/utils_spark.py

@@ -1,87 +0,0 @@
-import logging
-import os
-from typing import Dict
-from pyspark.sql import SparkSession
-from pyspark import SparkConf
-from pyspark.sql import types as T
-from pyspark.context import SparkContext
-
-
-logger = logging.getLogger(__name__)
-
-
-class UtilsSpark:
-    """class with handy functions to work with spark"""
-
-    __ENV_KEY_PYSPARK_IS_LOCAL = "PYSPARK_IS_LOCAL"
-    __ENV_KEY_NOT_AWS_CLOUD = "NOT_AWS_CLOUD"
-    __DTYPES_MAP = {
-        "bigint": T.LongType,
-        "string": T.StringType,
-        "double": T.DoubleType,
-        "int": T.IntegerType,
-        "boolean": T.BooleanType,
-        "timestamp": T.TimestampType,
-        "date": T.DateType,
-    }
-
-    @staticmethod
-    def get_local_spark_session(config: dict = None) -> SparkSession:
-        logger.debug(f"[get_local_spark_session|in] ({config})")
-        os.environ["PYSPARK_SUBMIT_ARGS"] = "--packages io.delta:delta-core_2.12:2.3.0 pyspark-shell"
-        builder = (
-            SparkSession.builder.config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
-            .config("spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog")
-            .config("spark.driver.host", "localhost")
-        )
-
-        if config is not None:
-            for k, v in config.items():
-                builder.config(k, v)
-
-        spark = builder.getOrCreate()
-
-        logger.debug(f"[get_local_spark_session|out] => {spark}")
-        return spark
-
-    @staticmethod
-    def get_spark_session(config: dict = None) -> SparkSession:
-        logger.debug(f"[get_spark_session|in] ({config})")
-
-        if "1" == os.getenv(UtilsSpark.__ENV_KEY_PYSPARK_IS_LOCAL):
-            spark: SparkSession = UtilsSpark.get_local_spark_session(config)
-        else:
-            if "1" == os.getenv(UtilsSpark.__ENV_KEY_NOT_AWS_CLOUD):
-                active_session = SparkSession.getActiveSession()
-            else:
-                from awsglue.context import GlueContext
-
-                glueContext = GlueContext(SparkContext.getOrCreate())
-                active_session = glueContext.spark_session
-
-            spark_config = SparkConf()
-
-            if active_session is not None:
-                former_config = active_session.sparkContext.getConf().getAll()
-                for entry in former_config:
-                    spark_config.set(entry[0], entry[1])
-            if config is not None:
-                for k, v in config.items():
-                    spark_config.set(k, v)
-                spark: SparkSession = SparkSession.builder.config(conf=spark_config).getOrCreate()
-            else:
-                spark: SparkSession = SparkSession.builder.getOrCreate()
-
-        logger.debug(f"[get_spark_session|out] => {spark}")
-        return spark
-
-    @staticmethod
-    def build_schema_from_dtypes(dtypes_schema: Dict[str, str]) -> T.StructType:
-        logger.info(f"[build_schema_from_dtypes|in] ({dtypes_schema})")
-        result = T.StructType()
-        for field, dtype in dtypes_schema.items():
-            new_type = UtilsSpark.__DTYPES_MAP[dtype]
-            result.add(field, new_type(), True)
-
-        logger.info(f"[build_schema_from_dtypes|out] => {result}")
-        return result

tgedr/dataops/etl.py

@@ -1,112 +0,0 @@
-from abc import ABC, abstractmethod
-import inspect
-import logging
-from typing import Any, Dict, Optional
-
-
-logger = logging.getLogger(__name__)
-
-
-class EtlException(Exception):
-    pass
-
-
-"""
-ETL is an abstract base class that should be extended when you want to run an ETL-like task/job
-A subclass of ETL has extract, transform and load methods.
-
-The ETL class has static utility methods that serve as an outline for the class. Use example below:
-
-```python
-class MyEtl(Etl):
-    @Etl.inject_configuration
-    def extract(self, MY_PARAM) -> None:
-        # "MY_PARAM" should be supplied in 'configuration' dict otherwise an exception will be raised
-
-    @Etl.inject_configuration
-    def load(self, NOT_IN_CONFIG=123) -> None:
-        # If you try to inject a configuration key that is NOT on the configuration dictionary
-        # supplied to the constructor, it will not throw an error as long as you set a default
-        # value in the method you wish to decorate
-        assert NOT_IN_CONFIG == 123, "This will be ok"
-
-```
-"""
-
-
-class Etl(ABC):
-    def __init__(self, configuration: Optional[Dict[str, Any]] = None) -> None:
-        """Initialize a new instance of ETL.
-
-        Parameters
-        ----------
-        configuration : Dict[str, Any]
-            source for configuration injection
-        """
-        self._configuration = configuration
-
-    @abstractmethod
-    def extract(self) -> Any:
-        raise NotImplementedError()
-
-    @abstractmethod
-    def transform(self) -> Any:
-        raise NotImplementedError()
-
-    @abstractmethod
-    def load(self) -> Any:
-        raise NotImplementedError()
-
-    def validate_extract(self):
-        """
-        Optional extra checks for extract step.
-        """
-
-    def validate_transform(self):
-        """
-        Optional extra checks for transform step.
-        """
-
-    def run(self) -> Any:
-        logger.info("[run|in]")
-
-        self.extract()
-        self.validate_extract()
-
-        self.transform()
-        self.validate_transform()
-
-        result: Any = self.load()
-
-        logger.info(f"[run|out] => {result}")
-        return result
-
-    @staticmethod
-    def inject_configuration(f):
-        def decorator(self):
-            signature = inspect.signature(f)
-
-            missing_params = []
-            params = {}
-            for param in [parameter for parameter in signature.parameters if parameter != "self"]:
-                if signature.parameters[param].default != inspect._empty:
-                    params[param] = signature.parameters[param].default
-                else:
-                    params[param] = None
-                    if self._configuration is None or param not in self._configuration:
-                        missing_params.append(param)
-
-                if self._configuration is not None and param in self._configuration:
-                    params[param] = self._configuration[param]
-
-            if 0 < len(missing_params):
-                raise EtlException(
-                    f"{type(self).__name__}.{f.__name__}: missing required configuration parameters: {missing_params}"
-                )
-
-            return f(
-                self,
-                *[params[argument] for argument in params],
-            )
-
-        return decorator

tgedr/dataops/processor.py

@@ -1,27 +0,0 @@
-import abc
-from typing import Any, Dict, Optional
-
-
-class ProcessorException(Exception):
-    pass
-
-
-class ProcessorInterface(metaclass=abc.ABCMeta):
-    """
-    def process(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()
-    """
-
-    @classmethod
-    def __subclasshook__(cls, subclass):
-        return hasattr(subclass, "process") and callable(subclass.process) or NotImplemented
-
-
-@ProcessorInterface.register
-class Processor(abc.ABC):
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        self._config = config
-
-    @abc.abstractmethod
-    def process(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()

tgedr/dataops/sink/local_fs_file_sink.py

@@ -1,47 +0,0 @@
-import logging
-import os
-import shutil
-from typing import Any, Dict, Optional
-
-from tgedr.dataops.sink.sink import Sink, SinkException
-
-
-logger = logging.getLogger(__name__)
-
-
-class LocalFsFileSink(Sink):
-    """sink class used to save/persist an object/file to a local fs location"""
-
-    CONTEXT_SOURCE_PATH = "source"
-    CONTEXT_TARGET_PATH = "target"
-
-    def put(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        logger.info(f"[put|in] ({context})")
-
-        if self.CONTEXT_SOURCE_PATH not in context:
-            raise SinkException(f"you must provide context for {self.CONTEXT_SOURCE_PATH}")
-        if self.CONTEXT_TARGET_PATH not in context:
-            raise SinkException(f"you must provide context for {self.CONTEXT_TARGET_PATH}")
-
-        source = context[self.CONTEXT_SOURCE_PATH]
-        target = context[self.CONTEXT_TARGET_PATH]
-
-        shutil.copy(source, target)
-        logger.info("[put|out]")
-
-    def delete(self, context: Optional[Dict[str, Any]] = None):
-        logger.info(f"[delete|in] ({context})")
-
-        if self.CONTEXT_TARGET_PATH not in context:
-            raise SinkException(f"you must provide context for {self.CONTEXT_TARGET_PATH}")
-
-        target = context[self.CONTEXT_TARGET_PATH]
-
-        if os.path.isfile(target):
-            os.remove(target)
-        elif os.path.isdir(target):
-            shutil.rmtree(target)
-        else:
-            raise SinkException(f"[delete] is it a dir or a folder? {target}")
-
-        logger.info("[delete|out]")

tgedr/dataops/sink/sink.py

@@ -1,46 +0,0 @@
-import abc
-from typing import Any, Dict, Optional
-
-from tgedr.dataops.chain import Chain
-
-
-class SinkException(Exception):
-    pass
-
-
-class SinkInterface(metaclass=abc.ABCMeta):
-    """
-    def put(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()
-    """
-
-    @classmethod
-    def __subclasshook__(cls, subclass):
-        return (
-            hasattr(subclass, "put")
-            and callable(subclass.put)
-            and hasattr(subclass, "delete")
-            and callable(subclass.delete)
-        ) or NotImplemented
-
-
-@SinkInterface.register
-class Sink(abc.ABC):
-    """abstract class defining methods ('put' and 'delete') to manage persistence of data somewhere as defined by implementing classes"""
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        self._config = config
-
-    @abc.abstractmethod
-    def put(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        raise NotImplementedError()
-
-    @abc.abstractmethod
-    def delete(self, context: Optional[Dict[str, Any]] = None):
-        raise NotImplementedError()
-
-
-@SinkInterface.register
-class SinkChain(Chain, abc.ABC):
-    def execute(self, context: Optional[Dict[str, Any]] = None) -> Any:
-        return self.put(context=context)

tgedr/dataops/source/abstract_s3_file_source.py

@@ -1,43 +0,0 @@
-from abc import ABC
-import logging
-from typing import Any, Dict, List, Optional
-
-from tgedr.dataops.commons.s3_connector import S3Connector
-from tgedr.dataops.commons.utils_fs import process_s3_url
-from tgedr.dataops.source.source import Source, SourceException
-
-
-logger = logging.getLogger()
-
-
-class AbstractS3FileSource(Source, S3Connector, ABC):
-    """abstract class used to read file sources from s3"""
-
-    CONTEXT_KEY_URL = "url"
-    CONTEXT_KEY_SUFFIX = "suffix"
-
-    def __init__(self, config: Optional[Dict[str, Any]] = None):
-        Source.__init__(self, config=config)
-        S3Connector.__init__(self)
-
-    def list(self, context: Optional[Dict[str, Any]] = None) -> List[str]:
-        logger.info(f"[list|in] ({context})")
-
-        result: List[str] = []
-        if self.CONTEXT_KEY_URL not in context:
-            raise SourceException(f"you must provide context for {self.CONTEXT_KEY_URL}")
-
-        protocol, bucket, key = process_s3_url(context[self.CONTEXT_KEY_URL])
-
-        objs = self._client.list_objects_v2(Bucket=bucket, Prefix=key)
-        result = [
-            (protocol + bucket + "/" + entry["Key"]) for entry in objs["Contents"] if not (entry["Key"]).endswith("/")
-        ]
-
-        if self.CONTEXT_KEY_SUFFIX in context:
-            suffix: str = context[self.CONTEXT_KEY_SUFFIX]
-            result = [f for f in result if f.endswith(suffix)]
-
-        logger.debug(f"[list|out] => {result}")
-        logger.info(f"[list|out] => result len: {len(result)}")
-        return result