tgedr-dataops-abs 0.0.1__py3-none-any.whl

tgedr_dataops_abs/chain.py

@@ -0,0 +1,129 @@
+"""Chain abstractions for building processing pipelines.
+
+Provides abstract base classes and interfaces for chaining operations together
+in a chain of responsibility pattern, allowing sequential processing steps.
+"""
+
+import abc
+from typing import Any
+
+from tgedr_dataops_abs.processor import Processor
+
+
+class ChainException(Exception):
+    """Exception raised for chain-related errors."""
+
+
+class ChainInterface(metaclass=abc.ABCMeta):
+    """Interface for chain implementations.
+
+    Defines the contract for classes that implement next and execute operations.
+    """
+
+    @classmethod
+    def __subclasshook__(cls, subclass):  # noqa: ANN001, ANN206
+        """Check if a class implements the chain interface."""
+        return (
+            hasattr(subclass, "next")
+            and callable(subclass.next)
+            and hasattr(subclass, "execute")
+            and callable(subclass.execute)
+        ) or NotImplemented
+
+
+class ChainMixin(abc.ABC):
+    """Mixin providing chain functionality for sequential execution.
+
+    Implements the chain of responsibility pattern for processing operations.
+    """
+
+    def next(self, handler: "ChainMixin") -> "ChainMixin":
+        """Add the next handler in the chain.
+
+        Parameters
+        ----------
+        handler : ChainMixin
+            The next handler to add to the chain.
+
+        Returns
+        -------
+        ChainMixin
+            The current chain instance for method chaining.
+        """
+        if "_next" not in self.__dict__ or self._next is None:
+            self._next: "ChainMixin" = handler  # noqa: UP037
+        else:
+            self._next.next(handler)
+        return self
+
+    @abc.abstractmethod
+    def execute(self, context: dict[str, Any] | None = None) -> Any:
+        """Execute the operation in the chain.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context to pass through the chain.
+
+        Returns
+        -------
+        Any
+            Result of the execution.
+        """
+        raise NotImplementedError
+
+
+class ProcessorChainMixin(ChainMixin):
+    """Mixin that combines processor and chain capabilities.
+
+    Executes processor logic and passes control to the next handler in the chain.
+    """
+
+    def execute(self, context: dict[str, Any] | None = None) -> Any:
+        """Execute processor and continue to next handler.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context to process and pass to next handler.
+
+        Returns
+        -------
+        Any
+            Result from processing.
+        """
+        self.process(context=context)
+        if "_next" in self.__dict__ and self._next is not None:
+            self._next.execute(context=context)
+
+
+@ChainInterface.register
+class ProcessorChain(ProcessorChainMixin, Processor):
+    """Concrete processor that can be chained with other processors.
+
+    Combines ProcessorChainMixin and Processor for chainable processing.
+    """
+
+
+@ChainInterface.register
+class Chain(ChainMixin, abc.ABC):
+    """Abstract base class for chainable operations.
+
+    Extends ChainMixin to provide a base for custom chainable components.
+    """
+
+    @abc.abstractmethod
+    def execute(self, context: dict[str, Any] | None = None) -> Any:
+        """Execute the chain operation.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context to pass through execution.
+
+        Returns
+        -------
+        Any
+            Result of the execution.
+        """
+        raise NotImplementedError

tgedr_dataops_abs/etl.py

@@ -0,0 +1,145 @@
+"""ETL (Extract, Transform, Load) abstractions for data processing workflows.
+
+Provides abstract base classes for implementing ETL pipelines with configuration
+injection, validation hooks, and structured execution flow.
+
+Example:
+-------
+```python
+class MyEtl(Etl):
+    @Etl.inject_configuration
+    def extract(self, MY_PARAM) -> None:
+        # "MY_PARAM" should be supplied in 'configuration' dict
+        # otherwise an exception will be raised
+        pass
+
+    @Etl.inject_configuration
+    def load(self, NOT_IN_CONFIG=123) -> None:
+        # If you try to inject a configuration key that is NOT in the
+        # configuration dictionary supplied to the constructor, it will
+        # not throw an error as long as you set a default value
+        assert NOT_IN_CONFIG == 123, "This will be ok"
+```
+"""
+
+from abc import ABC, abstractmethod
+import inspect
+import logging
+from typing import Any
+
+
+logger = logging.getLogger(__name__)
+
+
+class EtlException(Exception):
+    """Exception raised for ETL-related errors."""
+
+
+class Etl(ABC):
+    """Abstract base class for ETL (Extract, Transform, Load) operations.
+
+    Provides a template method pattern for ETL workflows with configuration
+    injection and optional validation hooks.
+    """
+
+    def __init__(self, configuration: dict[str, Any] | None = None) -> None:
+        """Initialize a new instance of ETL.
+
+        Parameters
+        ----------
+        configuration : dict[str, Any]
+            source for configuration injection
+        """
+        self._configuration = configuration
+
+    @abstractmethod
+    def extract(self) -> Any:
+        """Extract data from source.
+
+        Returns
+        -------
+        Any
+            Extracted data.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def transform(self) -> Any:
+        """Transform extracted data.
+
+        Returns
+        -------
+        Any
+            Transformed data.
+        """
+        raise NotImplementedError
+
+    @abstractmethod
+    def load(self) -> Any:
+        """Load transformed data to destination.
+
+        Returns
+        -------
+        Any
+            Result of load operation.
+        """
+        raise NotImplementedError
+
+    def validate_extract(self) -> None:  # noqa: B027
+        """Optional extra checks for extract step."""
+
+    def validate_transform(self) -> None:  # noqa: B027
+        """Optional extra checks for transform step."""
+
+    def run(self) -> Any:
+        """Execute the complete ETL workflow.
+
+        Runs extract, validate_extract, transform, validate_transform, and load
+        in sequence with structured logging.
+
+        Returns
+        -------
+        Any
+            Result from the load operation.
+        """
+        logger.info("[run|in]")
+
+        self.extract()
+        self.validate_extract()
+
+        self.transform()
+        self.validate_transform()
+
+        result: Any = self.load()
+
+        logger.info("[run|out] => %s", result)
+        return result
+
+    @staticmethod
+    def inject_configuration(f):  # noqa: ANN001, ANN205, D102
+        def decorator(self):  # noqa: ANN001, ANN202
+            signature = inspect.signature(f)
+
+            missing_params = []
+            params = {}
+            for param in [parameter for parameter in signature.parameters if parameter != "self"]:
+                if signature.parameters[param].default != inspect._empty:  # noqa: SLF001
+                    params[param] = signature.parameters[param].default
+                else:
+                    params[param] = None
+                    if self._configuration is None or param not in self._configuration:
+                        missing_params.append(param)
+
+                if self._configuration is not None and param in self._configuration:
+                    params[param] = self._configuration[param]
+
+            if 0 < len(missing_params):
+                msg = f"missing required configuration parameters: {missing_params}"
+                raise EtlException(msg)
+
+            return f(
+                self,
+                *[params[argument] for argument in params],
+            )
+
+        return decorator

tgedr_dataops_abs/processor.py

@@ -0,0 +1,55 @@
+"""Processor abstractions for data transformation operations.
+
+Provides abstract base classes and interfaces for implementing processors that
+transform data based on context.
+"""
+
+import abc
+from typing import Any
+
+
+class ProcessorException(Exception):
+    """Exception raised for processor-related errors."""
+
+
+class ProcessorInterface(metaclass=abc.ABCMeta):
+    """Interface for processor implementations.
+
+    Defines the contract for classes that implement process operations.
+    """
+
+    @classmethod
+    def __subclasshook__(cls, subclass):  # noqa: ANN001, ANN206
+        """Check if a class implements the processor interface."""
+        return (hasattr(subclass, "process") and callable(subclass.process)) or NotImplemented
+
+
+@ProcessorInterface.register
+class Processor(abc.ABC):
+    """Abstract base class for processors that transform data."""
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize processor with optional configuration.
+
+        Parameters
+        ----------
+        config : dict[str, Any] | None
+            Configuration dictionary for the processor.
+        """
+        self._config = config
+
+    @abc.abstractmethod
+    def process(self, context: dict[str, Any] | None = None) -> Any:
+        """Process data based on the provided context.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context containing data and parameters for processing.
+
+        Returns
+        -------
+        Any
+            Processed result.
+        """
+        raise NotImplementedError

tgedr_dataops_abs/sink.py

@@ -0,0 +1,96 @@
+"""Sink abstractions for write-only data persistence operations.
+
+Provides abstract base classes and interfaces for implementing sinks that handle
+data output operations (put, delete) without read capabilities.
+"""
+
+import abc
+from typing import Any
+
+from tgedr_dataops_abs.chain import Chain
+
+
+class SinkException(Exception):
+    """Exception raised for sink-related errors."""
+
+
+class SinkInterface(metaclass=abc.ABCMeta):
+    """Interface for sink implementations.
+
+    Defines the contract for classes that implement put and delete operations.
+    """
+
+    @classmethod
+    def __subclasshook__(cls, subclass):  # noqa: ANN001, ANN206
+        """Check if a class implements the sink interface."""
+        return (
+            hasattr(subclass, "put")
+            and callable(subclass.put)
+            and hasattr(subclass, "delete")
+            and callable(subclass.delete)
+        ) or NotImplemented
+
+
+@SinkInterface.register
+class Sink(abc.ABC):
+    """Abstract class defining methods ('put' and 'delete') to manage persistence of data somewhere as defined by implementing classes."""
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize sink with optional configuration.
+
+        Parameters
+        ----------
+        config : dict[str, Any] | None
+            Configuration dictionary for the sink.
+        """
+        self._config = config
+
+    @abc.abstractmethod
+    def put(self, context: dict[str, Any] | None = None) -> Any:
+        """Put data to the sink.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context containing data and metadata for the put operation.
+
+        Returns
+        -------
+        Any
+            Result of the put operation.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, context: dict[str, Any] | None = None) -> None:
+        """Delete data from the sink.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context containing information about what to delete.
+        """
+        raise NotImplementedError
+
+
+@SinkInterface.register
+class SinkChain(Chain, abc.ABC):
+    """Abstract sink that can be chained with other operations.
+
+    Combines Chain and Sink capabilities for building processing pipelines.
+    """
+
+    def execute(self, context: dict[str, Any] | None = None) -> Any:
+        """Execute the sink operation by calling put.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context to pass to the put operation.
+
+        Returns
+        -------
+        Any
+            Result of the put operation.
+        """
+        return self.put(context=context)

tgedr_dataops_abs/source.py

@@ -0,0 +1,105 @@
+"""Source abstractions for read-only data retrieval operations.
+
+Provides abstract base classes and interfaces for implementing sources that handle
+data input operations (get, list) without write capabilities.
+"""
+
+import abc
+from typing import Any
+
+from tgedr_dataops_abs.chain import Chain
+
+
+class SourceException(Exception):
+    """Exception raised for source-related errors."""
+
+
+class NoSourceException(SourceException):
+    """Exception raised when a requested source is not found."""
+
+
+class SourceInterface(metaclass=abc.ABCMeta):
+    """Interface for source implementations.
+
+    Defines the contract for classes that implement get and list operations.
+    """
+
+    @classmethod
+    def __subclasshook__(cls, subclass):  # noqa: ANN001, ANN206
+        """Check if a class implements the source interface."""
+        return (
+            hasattr(subclass, "get")
+            and callable(subclass.get)
+            and hasattr(subclass, "list")
+            and callable(subclass.list)
+        ) or NotImplemented
+
+
+@SourceInterface.register
+class Source(abc.ABC):
+    """Abstract class defining methods ('list' and 'get') to manage retrieval of data from somewhere as defined by implementing classes."""
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize source with optional configuration.
+
+        Parameters
+        ----------
+        config : dict[str, Any] | None
+            Configuration dictionary for the source.
+        """
+        self._config = config
+
+    @abc.abstractmethod
+    def get(self, context: dict[str, Any] | None = None) -> Any:
+        """Get data from the source.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context containing parameters for the get operation.
+
+        Returns
+        -------
+        Any
+            Retrieved data.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def list(self, context: dict[str, Any] | None = None) -> Any:
+        """List available items in the source.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context containing parameters for the list operation.
+
+        Returns
+        -------
+        Any
+            List of available items.
+        """
+        raise NotImplementedError
+
+
+@SourceInterface.register
+class SourceChain(Chain, abc.ABC):
+    """Abstract source that can be chained with other operations.
+
+    Combines Chain and Source capabilities for building processing pipelines.
+    """
+
+    def execute(self, context: dict[str, Any] | None = None) -> Any:
+        """Execute the source operation by calling get.
+
+        Parameters
+        ----------
+        context : dict[str, Any] | None
+            Context to pass to the get operation.
+
+        Returns
+        -------
+        Any
+            Result of the get operation.
+        """
+        return self.get(context=context)

tgedr_dataops_abs/store.py

@@ -0,0 +1,113 @@
+"""Store abstractions for CRUD (Create, Read, Update, Delete) operations.
+
+Provides abstract base classes and interfaces for implementing stores that handle
+full data persistence operations including get, save, update, and delete.
+"""
+
+import abc
+from typing import Any
+
+
+class StoreException(Exception):
+    """Exception raised for store-related errors."""
+
+
+class NoStoreException(StoreException):
+    """Exception raised when a requested store item is not found."""
+
+
+class StoreInterface(metaclass=abc.ABCMeta):
+    """Interface for store implementations.
+
+    Defines the contract for classes that implement CRUD operations.
+    """
+
+    @classmethod
+    def __subclasshook__(cls, subclass):  # noqa: ANN001, ANN206
+        """Check if a class implements the store interface."""
+        return (
+            hasattr(subclass, "get")
+            and callable(subclass.get)
+            and hasattr(subclass, "delete")
+            and callable(subclass.delete)
+            and hasattr(subclass, "save")
+            and callable(subclass.save)
+            and hasattr(subclass, "update")
+            and callable(subclass.update)
+        ) or NotImplemented
+
+
+@StoreInterface.register
+class Store(abc.ABC):
+    """Abstract class used to manage persistence, defining CRUD-like (CreateReadUpdateDelete) methods."""
+
+    def __init__(self, config: dict[str, Any] | None = None) -> None:
+        """Initialize store with optional configuration.
+
+        Parameters
+        ----------
+        config : dict[str, Any] | None
+            Configuration dictionary for the store.
+        """
+        self._config = config
+
+    @abc.abstractmethod
+    def get(self, key: str, **kwargs) -> Any:  # noqa: ANN003
+        """Get data from the store by key.
+
+        Parameters
+        ----------
+        key : str
+            The key identifying the data to retrieve.
+        **kwargs
+            Additional store-specific parameters.
+
+        Returns
+        -------
+        Any
+            Retrieved data.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def delete(self, key: str, **kwargs) -> None:  # noqa: ANN003
+        """Delete data from the store by key.
+
+        Parameters
+        ----------
+        key : str
+            The key identifying the data to delete.
+        **kwargs
+            Additional store-specific parameters.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def save(self, df: Any, key: str, **kwargs) -> Any:  # noqa: ANN003
+        """Save data to the store.
+
+        Parameters
+        ----------
+        df : Any
+            The data to save.
+        key : str
+            The key to associate with the data.
+        **kwargs
+            Additional store-specific parameters.
+        """
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def update(self, df: Any, key: str, **kwargs) -> Any:  # noqa: ANN003
+        """Update existing data in the store.
+
+        Parameters
+        ----------
+        df : Any
+            The updated data.
+        key : str
+            The key identifying the data to update.
+        **kwargs
+            Additional store-specific parameters.
+        """
+        raise NotImplementedError

tgedr_dataops_abs-0.0.1.dist-info/METADATA

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: tgedr-dataops-abs
+Version: 0.0.1
+Summary: this is a template for a python package
+Author-email: developer <developer@email.com>
+Requires-Python: >=3.11
+Description-Content-Type: text/markdown
+Requires-Dist: pandas>=2.3.0
+
+# tgedr-dataops-abs
+
+![Coverage](./coverage.svg)
+[![PyPI](https://img.shields.io/pypi/v/tgedr-dataops-abs)](https://pypi.org/project/tgedr-dataops-abs/)
+
+
+data operations related code - abstractions
+
+
+## motivation
+abstract constructs to depict commonly use cases in data engineering context. Think about commonly accepted abstract classes and interfaces that can be implemented and extended according to different requirements and constraints.
+
+
+## package namespaces and its contents
+
+- __Chain__ : chain-like abstract class (for sequential processing) ([example](tests/tgedr_dataops_abs/test_processor_chain.py))
+- __Etl__ : Extract-Transform-Load abstract class to be extended and used in data pipelines ([example](tests/tgedr_dataops_abs/test_etl.py))
+- __Processor__ : abstract class for data processing ([example](tests/tgedr_dataops_abs/test_processor_chain.py))
+- __Sink__: abstract **sink** class defining methods (`put`and `delete`) to manage persistence of data somewhere as defined by implementing classes  ([example](tests/tgedr_dataops_abs/test_sink.py))
+- __Source__: abstract **source** class defining methods (`list` and `get`) to manage retrieval of data from somewhere as defined by implementing classes ([example](tests/tgedr_dataops_abs/test_source.py))
+- __Store__ : abstract class used to manage persistence, defining CRUD-like (CreateReadUpdateDelete) methods ([example](tests/tgedr_dataops_abs/test_store.py))
+
+
+
+## development
+- main requirements:
+  - _uv_  
+  - _bash_
+- Clone the repository like this:
+
+  ``` bash
+  git clone git@github.com:tgedr/dataops-abs
+  ```
+- cd into the folder: `cd dataops-abs`
+- install requirements: `./helper.sh reqs`

tgedr_dataops_abs-0.0.1.dist-info/RECORD

@@ -0,0 +1,11 @@
+tgedr_dataops_abs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+tgedr_dataops_abs/chain.py,sha256=ucz7CvZ4R6v5jZRT9smcX1BHJ9fresraoqfr_kSRovk,3571
+tgedr_dataops_abs/etl.py,sha256=43lL-LPRdY8O2YDlowfvVXL5Kz5_EIWG93f3gzy479I,4129
+tgedr_dataops_abs/processor.py,sha256=a5Gv6RcxzoORLfLbL_KXZiogfXUSDjoP2IJJDNroT6Y,1582
+tgedr_dataops_abs/sink.py,sha256=kLNaui9v3fyx5qfLKCagBIp2G4xotPGwcfkVke354qo,2720
+tgedr_dataops_abs/source.py,sha256=oDd-Ov4R8ioWu3WRqdPsRCPgjjOZRMiT0_wLYGxRQ3o,2924
+tgedr_dataops_abs/store.py,sha256=u9BQrlBe4Qfx00QxynPtSZKlvOMez4nlNP0EpDVD6JY,3211
+tgedr_dataops_abs-0.0.1.dist-info/METADATA,sha256=eDNLfEhG6U3gUNx2m23dRPeTyD-FKwrG8G7LJmnzUtY,1926
+tgedr_dataops_abs-0.0.1.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
+tgedr_dataops_abs-0.0.1.dist-info/top_level.txt,sha256=VJEbLpMLOMqqPUh_BXrAxxV1LP0kBtXNS95AGxknAJ8,18
+tgedr_dataops_abs-0.0.1.dist-info/RECORD,,

tgedr_dataops_abs-0.0.1.dist-info/WHEEL

@@ -0,0 +1,5 @@
+Wheel-Version: 1.0
+Generator: setuptools (80.10.2)
+Root-Is-Purelib: true
+Tag: py3-none-any
+

tgedr_dataops_abs-0.0.1.dist-info/top_level.txt

@@ -0,0 +1 @@
+tgedr_dataops_abs