koheesio 0.7.0__py3-none-any.whl

koheesio/__about__.py

@@ -0,0 +1,48 @@
+"""A type-safe Python framework for building efficient data pipelines
+
+Koheesio is a Python framework for building type-safe and efficient data pipelines, particularly useful for
+large-scale data processing. It promotes modularity, composability, and collaboration, allowing for the creation of
+complex pipelines from simple, reusable components.
+
+Leveraging Pydantic, it ensures predictable pipeline execution and structured configurations within pipeline components.
+
+With its simple and straightforward API, Koheesio makes it easy for developers to build and manage data pipelines,
+enhancing productivity and code maintainability.
+"""
+
+LICENSE_INFO = "Licensed as Apache 2.0"
+SOURCE = "https://github.com/Nike-Inc/koheesio"
+__version__ = "0.7.0"
+__logo__ = 75, (
+    b"\x1f\x8b\x08\x00TiGf\x02\xff}\x91\xbb\r\xc30\x0cD{Nq\x1bh\n\x01\x16R\xa4pK@\x8bh\xf8\xe8\xf8\x89\xe9\x04\xf0\x15"
+    b"\xc4\x91\x10\x9f(J`z\xbd4B\xea8J\xf2\xa01T\x02\x01,\x0b\x85Q\x92\x07\xe9\x9cK\x92\xd1,\xe0mRBL\x9c\xa6\x9b\xee"
+    b"\xeet)\x07Av\xc9/\x0b\x98\x93\xb4=\xd1v\xa4\xf5NG\xc6\xe5\xce\x93nk\x8d\x81\xf5\xed\x92\x80AmC\xbb\xde,.\x7f\x1fc"
+    b"\x0fU\xa79\x19\x82\x16]\x1248\x8f\xa5\x7f\x1c|\x92\xe2\xb8\xa59\xfd\xa5\x86\x8b.I\x9a\xf3\xd4W\x80\x8a\xd3\x9e"
+    b"\xfb\xba\\\xecm\x9f#\xee\xea\x92}M+\xffb\xb7\xb2\xc4\xc4K\x88Zui\xda\xedD\xfb\x00\xcfU6\xd3_\x02\x00\x00"
+)
+__short_description__ = __doc__.split("\n", maxsplit=1)[0]
+__about__ = f"""Koheesio - v{__version__}
+{__short_description__}
+{LICENSE_INFO}
+Source: {SOURCE}
+"""
+
+
+# fmt: off
+def _about():  # pragma: no cover
+    """Return the Koheesio logo and version/about information as a string
+    Note: this code is not meant to be readable, instead it is written to be as compact as possible
+    """
+    import gzip
+    _w, _logo = __logo__
+    _h, _v, _n = "\u2500", "\u2502", "\n"
+    return gzip.decompress(_logo).decode() + (
+        _n + '\u256d' + _h * _w + "\u256e" + _n +
+        f"{_n.join([f'{_v}{_l:^{_w}}{_v}' for _l in __about__.split(_n)[:-1]])}" +
+        _n + "\u2570" + _h * _w + "\u256f"
+    )
+# fmt: on
+
+
+if __name__ == "__main__":
+    print(_about())

koheesio/__init__.py

@@ -0,0 +1,37 @@
+# pragma: no cover
+from os import environ
+
+from koheesio.__about__ import __version__, _about
+from koheesio.context import Context
+from koheesio.logger import LoggingFactory
+from koheesio.models import BaseModel, ExtraParamsMixin
+from koheesio.steps import Step, StepOutput
+from koheesio.utils import convert_str_to_bool
+
+_koheesio_print_logo = convert_str_to_bool(environ.get("KOHEESIO__PRINT_LOGO", "True"))
+_logo_printed = False
+ABOUT = _about()
+VERSION = __version__
+
+__all__ = [
+    "ABOUT",
+    "BaseModel",
+    "Context",
+    "ExtraParamsMixin",
+    "LoggingFactory",
+    "Step",
+    "StepOutput",
+    "VERSION",
+]
+
+
+def print_logo():
+    global _logo_printed
+    global _koheesio_print_logo
+
+    if not _logo_printed and _koheesio_print_logo:
+        print(ABOUT)
+        _logo_printed = True
+
+
+print_logo()

koheesio/asyncio/__init__.py

@@ -0,0 +1,107 @@
+"""
+This module provides classes for asynchronous steps in the koheesio package.
+"""
+
+from typing import Dict, Union
+from abc import ABC
+from asyncio import iscoroutine
+
+from koheesio.steps import Step, StepMetaClass, StepOutput
+
+
+class AsyncStepMetaClass(StepMetaClass):
+    """Metaclass for asynchronous steps.
+
+    This metaclass is used to define asynchronous steps in the Koheesio framework.
+    It inherits from the StepMetaClass and provides additional functionality for
+    executing asynchronous steps.
+
+    Attributes:
+        None
+
+    Methods:
+        _execute_wrapper: Wrapper method for executing asynchronous steps.
+
+    """
+
+    def _execute_wrapper(cls, *args, **kwargs):
+        """Wrapper method for executing asynchronous steps.
+
+        This method is called when an asynchronous step is executed. It wraps the
+        execution of the step with additional functionality.
+
+        Parameters
+        ----------
+            *args: Variable length argument list.
+            **kwargs: Arbitrary keyword arguments.
+
+        Returns
+        -------
+            The result of executing the asynchronous step.
+
+        """
+
+        return super()._execute_wrapper(*args, **kwargs)
+
+
+class AsyncStepOutput(Step.Output):
+    """
+    Represents the output of an asynchronous step.
+
+    This class extends the base `Step.Output` class and provides additional functionality for merging key-value maps.
+
+    Attributes
+    ----------
+    ...
+
+    Methods
+    -------
+    merge(other: Union[Dict, StepOutput])
+        Merge key-value map with self.
+    """
+
+    def merge(self, other: Union[Dict, StepOutput]):
+        """Merge key,value map with self
+
+        Examples
+        --------
+        ```python
+        step_output = StepOutput(foo="bar")
+        step_output.merge({"lorem": "ipsum"})  # step_output will now contain {'foo': 'bar', 'lorem': 'ipsum'}
+        ```
+
+        Functionally similar to adding two dicts together; like running `{**dict_a, **dict_b}`.
+
+        Parameters
+        ----------
+        other: Union[Dict, StepOutput]
+            Dict or another instance of a StepOutputs class that will be added to self
+        """
+        if isinstance(other, StepOutput):
+            other = other.model_dump()  # ensures we really have a dict
+
+        if not iscoroutine(other):
+            for k, v in other.items():
+                self.set(k, v)
+
+        return self
+
+
+class AsyncStep(Step, ABC, metaclass=AsyncStepMetaClass):
+    """
+    Asynchronous step class that inherits from Step and uses the AsyncStepMetaClass metaclass.
+
+    Attributes
+    ----------
+    Output : AsyncStepOutput
+        The output class for the asynchronous step.
+    """
+
+    class Output(AsyncStepOutput):
+        """
+        Output class for asyncio step.
+
+        This class represents the output of the asyncio step. It inherits from the AsyncStepOutput class.
+        """
+
+    __output__: Output

koheesio/asyncio/http.py

@@ -0,0 +1,383 @@
+"""
+This module contains async implementation of HTTP step.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import warnings
+from typing import Any, Dict, List, Optional, Tuple, Union
+
+import nest_asyncio
+import yarl
+from aiohttp import BaseConnector, ClientSession, TCPConnector
+from aiohttp_retry import ExponentialRetry, RetryClient, RetryOptionsBase
+
+from pydantic import Field, SecretStr, field_validator, model_validator
+
+from koheesio.asyncio import AsyncStep, AsyncStepOutput
+from koheesio.models import ExtraParamsMixin
+from koheesio.steps.http import HttpMethod
+
+
+class AsyncHttpStep(AsyncStep, ExtraParamsMixin):
+    """
+    Asynchronous HTTP step for making HTTP requests using aiohttp.
+
+    Parameters
+    ----------
+    client_session : Optional[ClientSession]
+        Aiohttp ClientSession.
+    url : List[yarl.URL]
+        List of yarl.URL.
+    retry_options : Optional[RetryOptionsBase]
+        Retry options for the request.
+    connector : Optional[BaseConnector]
+        Connector for the aiohttp request.
+    headers : Optional[Dict[str, Union[str, SecretStr]]]
+        Request headers.
+
+    Output
+    ------
+    responses_urls : Optional[List[Tuple[Dict[str, Any], yarl.URL]]]
+        List of responses from the API and request URL.
+
+    Examples
+    --------
+    ```python
+    >>> import asyncio
+    >>> from aiohttp import ClientSession
+    >>> from aiohttp.connector import TCPConnector
+    >>> from aiohttp_retry import ExponentialRetry
+    >>> from koheesio.steps.async.http import AsyncHttpStep
+    >>> from yarl import URL
+    >>> from typing import Dict, Any, Union, List, Tuple
+    >>>
+    >>> # Initialize the AsyncHttpStep
+    >>> async def main():
+    >>>     session = ClientSession()
+    >>>     urls = [URL('https://example.com/api/1'), URL('https://example.com/api/2')]
+    >>>     retry_options = ExponentialRetry()
+    >>>     connector = TCPConnector(limit=10)
+    >>>     headers = {'Content-Type': 'application/json'}
+    >>>     step = AsyncHttpStep(
+    >>>         client_session=session,
+    >>>         url=urls,
+    >>>         retry_options=retry_options,
+    >>>         connector=connector,
+    >>>         headers=headers
+    >>>     )
+    >>>
+    >>>     # Execute the step
+    >>>     responses_urls=  await step.get()
+    >>>
+    >>>     return responses_urls
+    >>>
+    >>> # Run the main function
+    >>> responses_urls = asyncio.run(main())
+    ```
+    """
+
+    client_session: Optional[ClientSession] = Field(default=None, description="Aiohttp ClientSession", exclude=True)
+    url: List[yarl.URL] = Field(
+        default=None,
+        alias="urls",
+        description="""Expecting list, as there is no value in executing async request for one value.
+        yarl.URL is preferable, because params/data can be injected into URL instance""",
+        exclude=True,
+    )
+    retry_options: Optional[RetryOptionsBase] = Field(
+        default=None, description="Retry options for the request", exclude=True
+    )
+    connector: Optional[BaseConnector] = Field(
+        default=None, description="Connector for the aiohttp request", exclude=True
+    )
+    headers: Dict[str, Union[str, SecretStr]] = Field(
+        default_factory=dict,
+        description="Request headers",
+        alias="header",
+        exclude=True,
+    )
+
+    timeout: None = Field(default=None, description="[Optional] Request timeout")
+
+    method: Union[str, HttpMethod] = Field(
+        default=HttpMethod.GET,
+        description="What type of Http call to perform. One of 'get', 'post', 'put', 'delete'. Defaults to 'get'.",
+    )
+
+    class Output(AsyncStepOutput):
+        """Output class for Step"""
+
+        responses_urls: Optional[List[Tuple[Dict[str, Any], yarl.URL]]] = Field(
+            default=None, description="List of responses from the API and request URL", repr=False
+        )
+
+    def __tasks_generator(self, method) -> List[asyncio.Task]:
+        """
+        Generate a list of tasks for making HTTP requests.
+
+        Parameters
+        ----------
+        method : HttpMethod
+            The HTTP method to use for the requests.
+
+        Returns
+        -------
+        List[Task]
+            A list of asyncio tasks for making HTTP requests.
+        """
+        tasks = [
+            asyncio.create_task(
+                self.request(
+                    method=method,
+                    url=u,
+                    headers=self.get_headers(),
+                )
+            )
+            for u in self.url
+        ]
+
+        return tasks
+
+    @model_validator(mode="after")
+    def _move_extra_params_to_params(self):
+        """
+        Move extra_params to params dict.
+
+        Returns
+        -------
+        AsyncHttpStep
+            The updated AsyncHttpStep instance.
+        """
+        return self
+
+    async def _execute(self, tasks: List[asyncio.Task]) -> List[Tuple[Dict[str, Any], yarl.URL]]:
+        """
+        Execute the HTTP requests asynchronously.
+
+        Parameters
+        ----------
+        tasks : List[Task]
+            A list of asyncio tasks for making HTTP requests.
+
+        Returns
+        -------
+        List[Tuple[Dict[str, Any], yarl.URL]]
+            A list of response data and corresponding request URLs.
+        """
+        self._init_session()
+        try:
+            responses_urls = await asyncio.gather(*tasks)
+        finally:
+            await self.client_session.close()
+            await self.__retry_client.close()
+
+        return responses_urls
+
+    def _init_session(self):
+        """
+        Initialize the aiohttp session and retry client.
+        """
+        self.connector = self.connector or TCPConnector(limit=10)
+        self.client_session = self.client_session or ClientSession(connector=self.connector)
+        self.retry_options = self.retry_options or ExponentialRetry()
+        # Disable pylint warning: attribute is not initialized in __init__
+        # pylint: disable=W0201
+        self.__retry_client = RetryClient(
+            client_session=self.client_session, logger=self.log, retry_options=self.retry_options, raise_for_status=True
+        )
+
+    @field_validator("timeout")
+    def validate_timeout(cls, timeout):
+        """
+        Validate the 'data' field.
+
+        Parameters
+        ----------
+        data : Any
+            The value of the 'timeout' field.
+
+        Raises
+        ------
+        ValueError
+            If 'data' is not allowed in AsyncHttpStep.
+        """
+        if timeout:
+            raise ValueError("timeout is not allowed in AsyncHttpStep. Provide timeout through retry_options.")
+
+    def get_headers(self):
+        """
+        Get the request headers.
+
+        Returns
+        -------
+        Optional[Dict[str, Union[str, SecretStr]]]
+            The request headers.
+        """
+        _headers = None
+
+        if self.headers:
+            _headers = {k: v.get_secret_value() if isinstance(v, SecretStr) else v for k, v in self.headers.items()}
+
+            for k, v in self.headers.items():
+                if isinstance(v, SecretStr):
+                    self.headers[k] = v.get_secret_value()
+
+        return _headers or self.headers
+
+    def set_outputs(self, response):
+        """
+        Set the outputs of the step.
+
+        Parameters
+        ----------
+        response : Any
+            The response data.
+        """
+        warnings.warn("set outputs is not implemented in AsyncHttpStep.")
+
+    def get_options(self):
+        """
+        Get the options of the step.
+        """
+        warnings.warn("get_options is not implemented in AsyncHttpStep.")
+
+    # Disable pylint warning: method was expected to be 'non-async'
+    # pylint: disable=W0236
+    async def request(
+        self,
+        method: HttpMethod,
+        url: yarl.URL,
+        **kwargs,
+    ) -> Tuple[Dict[str, Any], yarl.URL]:
+        """
+        Make an HTTP request.
+
+        Parameters
+        ----------
+        method : HttpMethod
+            The HTTP method to use for the request.
+        url : yarl.URL
+            The URL to make the request to.
+        kwargs : Any
+            Additional keyword arguments to pass to the request.
+
+        Returns
+        -------
+        Tuple[Dict[str, Any], yarl.URL]
+            A tuple containing the response data and the request URL.
+        """
+        async with self.__retry_client.request(method=method, url=url, **kwargs) as response:
+            res = await response.json()
+
+        return (res, response.request_info.url)
+
+    # Disable pylint warning: method was expected to be 'non-async'
+    # pylint: disable=W0236
+    async def get(self) -> List[Tuple[Dict[str, Any], yarl.URL]]:
+        """
+        Make GET requests.
+
+        Returns
+        -------
+        List[Tuple[Dict[str, Any], yarl.URL]]
+            A list of response data and corresponding request URLs.
+        """
+        tasks = self.__tasks_generator(method=HttpMethod.GET)
+        responses_urls = await self._execute(tasks=tasks)
+
+        return responses_urls
+
+    # Disable pylint warning: method was expected to be 'non-async'
+    # pylint: disable=W0236
+    async def post(self) -> List[Tuple[Dict[str, Any], yarl.URL]]:
+        """
+        Make POST requests.
+
+        Returns
+        -------
+        List[Tuple[Dict[str, Any], yarl.URL]]
+            A list of response data and corresponding request URLs.
+        """
+        tasks = self.__tasks_generator(method=HttpMethod.POST)
+        responses_urls = await self._execute(tasks=tasks)
+
+        return responses_urls
+
+    # Disable pylint warning: method was expected to be 'non-async'
+    # pylint: disable=W0236
+    async def put(self) -> List[Tuple[Dict[str, Any], yarl.URL]]:
+        """
+        Make PUT requests.
+
+        Returns
+        -------
+        List[Tuple[Dict[str, Any], yarl.URL]]
+            A list of response data and corresponding request URLs.
+        """
+        tasks = self.__tasks_generator(method=HttpMethod.PUT)
+        responses_urls = await self._execute(tasks=tasks)
+
+        return responses_urls
+
+    # Disable pylint warning: method was expected to be 'non-async'
+    # pylint: disable=W0236
+    async def delete(self) -> List[Tuple[Dict[str, Any], yarl.URL]]:
+        """
+        Make DELETE requests.
+
+        Returns
+        -------
+        List[Tuple[Dict[str, Any], yarl.URL]]
+            A list of response data and corresponding request URLs.
+        """
+        tasks = self.__tasks_generator(method=HttpMethod.DELETE)
+        responses_urls = await self._execute(tasks=tasks)
+
+        return responses_urls
+
+    def execute(self) -> AsyncHttpStep.Output:
+        """
+        Execute the step.
+
+        Raises
+        ------
+        ValueError
+            If the specified HTTP method is not implemented in AsyncHttpStep.
+        """
+        # By design asyncio does not allow its event loop to be nested. This presents a practical problem:
+        #   When in an environment where the event loop is already running
+        #   it’s impossible to run tasks and wait for the result.
+        #   Trying to do so will give the error “RuntimeError: This event loop is already running”.
+        #   The issue pops up in various environments, such as web servers, GUI applications and in
+        #   Jupyter/DataBricks notebooks.
+        nest_asyncio.apply()
+
+        map_method_func = {
+            HttpMethod.GET: self.get,
+            HttpMethod.POST: self.post,
+            HttpMethod.PUT: self.put,
+            HttpMethod.DELETE: self.delete,
+        }
+
+        if self.method not in map_method_func:
+            raise ValueError(f"Method {self.method} not implemented in AsyncHttpStep.")
+
+        self.output.responses_urls = asyncio.run(map_method_func[self.method]())
+
+        return self.output
+
+
+class AsyncHttpGetStep(AsyncHttpStep):
+    """
+    Represents an asynchronous HTTP GET step.
+
+    This class inherits from the AsyncHttpStep class and specifies the HTTP method as GET.
+
+    Attributes:
+        method (HttpMethod): The HTTP method for the step, set to HttpMethod.GET.
+
+    """
+
+    method: HttpMethod = HttpMethod.GET