flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/CopyToRethink.py

@@ -0,0 +1,207 @@
+import asyncio
+from collections.abc import Callable
+import math
+import pandas as pd
+from asyncdb import AsyncDB
+from asyncdb.exceptions import (
+    StatementError,
+    DataError
+)
+from .CopyTo import CopyTo
+from ..interfaces.dataframes import PandasDataframe
+from ..exceptions import (
+    ComponentError,
+    DataNotFound
+)
+from ..conf import (
+    RT_DRIVER,
+    RT_HOST,
+    RT_PORT,
+    RT_USER,
+    RT_PASSWORD,
+    RT_DATABASE
+)
+
+
+class CopyToRethink(CopyTo, PandasDataframe):
+    """
+    CopyToRethink.
+
+    Overview
+
+        This component allows copy data into a RethinkDB table,
+        Copy into main rethinkdb using write functionality.
+
+    .. table:: Properties
+       :widths: auto
+
+
+    +--------------+----------+-----------+--------------------------------------------+
+    | Name         | Required | Summary                                                |
+    +--------------+----------+-----------+--------------------------------------------+
+    | tablename    |   Yes    | Name of the table in                                   |
+    |              |          | the database                                           |
+    +--------------+----------+-----------+--------------------------------------------+
+    | schema       |   Yes    | Name of the schema                                     |
+    |              |          | where is to the table, alias: database                 |
+    +--------------+----------+-----------+--------------------------------------------+
+    | truncate     |   Yes    | This option indicates if the component should empty    |
+    |              |          | before coping the new data to the table. If set to true|
+    |              |          | the table will be truncated before saving the new data.|
+    +--------------+----------+-----------+--------------------------------------------+
+    | use_buffer   |   No     | When activated, this option allows optimizing the      |
+    |              |          | performance of the task, when dealing with large       |
+    |              |          | volumes of data.                                       |
+    +--------------+----------+-----------+--------------------------------------------+
+    | credentials  |   No     | Supporting manual rethinkdb credentials                |
+    |              |          |                                                        |
+    +--------------+----------+-----------+--------------------------------------------+
+    | datasource   |   No     | Using a Datasource instead manual credentials          |
+    |              |          |                                                        |
+    +--------------+----------+-----------+--------------------------------------------+
+    
+
+        Example:
+
+        ```yaml
+        CopyToRethink:
+          tablename: product_availability
+          schema: bose
+        ```
+
+    """
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.pk = []
+        self.truncate: bool = False
+        self.data = None
+        self._engine = None
+        self.tablename: str = ""
+        self.schema: str = ""
+        self.use_chunks = False
+        self._chunksize: int = kwargs.pop('chunksize', 10)
+        self._connection: Callable = None
+        try:
+            self.multi = bool(kwargs["multi"])
+            del kwargs["multi"]
+        except KeyError:
+            self.multi = False
+        super().__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        self._driver: str = RT_DRIVER
+
+    def default_connection(self):
+        """default_connection.
+
+        Default Connection to RethinkDB.
+        """
+        try:
+            kwargs: dict = {
+                "host": RT_HOST,
+                "port": int(RT_PORT),
+                "db": RT_DATABASE,
+                "user": RT_USER,
+                "password": RT_PASSWORD
+            }
+            self._connection = AsyncDB(
+                RT_DRIVER,
+                params=kwargs,
+                loop=self._loop,
+                **kwargs
+            )
+            return self._connection
+        except Exception as err:
+            raise ComponentError(
+                f"Error configuring Pg Connection: {err!s}"
+            ) from err
+
+    # Function to clean invalid float values
+    def clean_floats(self, data):
+        def sanitize_value(value):
+            if isinstance(value, float) and (math.isnan(value) or math.isinf(value)):
+                return None  # Replace invalid floats with None or other default
+            return value
+
+        if isinstance(data, dict):
+            return {k: sanitize_value(v) for k, v in data.items()}
+        elif isinstance(data, list):
+            return [self.clean_floats(item) for item in data]
+        return data
+
+    async def _create_table(self):
+        # Create a Table using Model
+        async with await self._connection.connection() as conn:
+            await conn.use(self.schema)
+            await self._connection.create_table(
+                table=self.tablename
+            )
+
+    async def _truncate_table(self):
+        async with await self._connection.connection() as conn:
+            await conn.use(self.schema)
+            await self._connection.delete(
+                table=self.tablename
+            )
+
+    async def _copy_dataframe(self):
+        # saving directly the dataframe with write
+        try:
+            # can remove NAT from str fields:
+            u = self.data.select_dtypes(include=["string"])
+            if not u.empty:
+                self.data[u.columns] = u.astype(object).where(
+                    pd.notnull(u), None
+                )
+            # Convert the DataFrame to list of dictionaries and clean it
+            data_records = self.data.to_dict(orient='records')
+            cleaned_data = self.clean_floats(data_records)
+            async with await self._connection.connection() as conn:
+                await conn.use(self.schema)
+                await self._connection.write(
+                    table=self.tablename,
+                    data=cleaned_data,
+                    batch_size=self._chunksize,
+                    on_conflict="replace",
+                    changes=True,
+                    durability="soft",
+                )
+        except StatementError as err:
+            raise ComponentError(
+                f"Statement error: {err}"
+            ) from err
+        except DataError as err:
+            raise ComponentError(
+                f"Data error: {err}"
+            ) from err
+        except Exception as err:
+            raise ComponentError(
+                f"{self.StepName} Error: {err!s}"
+            ) from err
+
+    async def _copy_iterable(self):
+        """Copy an iterable to RethinkDB."""
+        try:
+            async with await self._connection.connection() as conn:
+                await conn.use(self.schema)
+                await conn.write(
+                    data=self.data,
+                    table=self.tablename,
+                    database=self.schema,
+                    batch_size=self._chunksize,
+                    on_conflict="replace",
+                    changes=True,
+                    durability="soft",
+                )
+        except Exception as err:
+            raise ComponentError(
+                f"Error copying iterable to RethinkDB: {err}"
+            ) from err

flowtask/components/CreateGCSBucket.py

@@ -0,0 +1,102 @@
+import asyncio
+from typing import Callable, Tuple
+from asyncdb import AsyncDB
+from querysource.datasources.drivers.bigquery import bigquery_default
+from .flow import FlowComponent
+from ..exceptions import ComponentError
+
+
+class CreateGCSBucket(FlowComponent):
+    """
+    CreateGCSBucket.
+
+    Este componente crea un bucket en Google Cloud Storage (GCS).
+
+    Properties:
+
+    - bucket_name: Nombre único para el bucket de GCS. (Requerido)
+    - location: Región geográfica donde se creará el bucket. (Opcional, por defecto 'US')
+    - storage_class: Clase de almacenamiento del bucket. (Opcional, por defecto 'STANDARD')
+    - overwrite: Permite proceder si el bucket ya existe. (Opcional, por defecto 'False')
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.bucket_name: str = kwargs.pop('bucket_name')
+        self.location: str = kwargs.pop('location', 'US')
+        self.storage_class: str = kwargs.pop('storage_class', 'STANDARD')
+        self.overwrite: bool = kwargs.pop('overwrite', False)
+        self.bq = None  # Instancia de AsyncDB
+        super(CreateGCSBucket, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Inicializa el componente configurando la conexión AsyncDB."""
+        # Validar parámetro requerido
+        if not self.bucket_name:
+            raise ComponentError("CreateGCSBucket: 'bucket_name' es un parámetro requerido.")
+
+        if not bigquery_default:
+            raise ComponentError("CreateGCSBucket: 'bigquery_default' no está configurado correctamente.")
+
+        # Obtener credenciales y parámetros del driver
+        credentials = bigquery_default.get_credentials()
+
+        # Inicializar AsyncDB con el driver de BigQuery
+        try:
+            self.bq = AsyncDB("bigquery", params=credentials)
+            self._logger.info("CreateGCSBucket: Instancia de AsyncDB creada exitosamente.")
+        except Exception as e:
+            raise ComponentError(f"CreateGCSBucket: Error al inicializar AsyncDB: {e}") from e
+
+    async def run(self) -> Tuple[str, str]:
+        """Ejecuta la creación del bucket en GCS."""
+        if not self.bq:
+            raise ComponentError("CreateGCSBucket: AsyncDB no está inicializado. Asegúrate de ejecutar 'start' antes de 'run'.")
+
+        try:
+            async with await self.bq.connection() as conn:
+                # Verificar si el bucket existe
+                bucket_exists = await conn.bucket_exists(self.bucket_name)
+                if not bucket_exists:
+                    # Crear el bucket
+                    await conn.create_bucket(
+                        bucket_name=self.bucket_name,
+                        location=self.location,
+                        storage_class=self.storage_class
+                    )
+                    message = f"Bucket '{self.bucket_name}' creado exitosamente en la región '{self.location}' con clase de almacenamiento '{self.storage_class}'."
+                    self._logger.info(message)
+                else:
+                    if self.overwrite:
+                        message = f"CreateGCSBucket: El bucket '{self.bucket_name}' ya existe."
+                        self._logger.warning(message)
+                    else:
+                        raise ComponentError(f"CreateGCSBucket: El bucket '{self.bucket_name}' ya existe y 'overwrite' está establecido en False.")
+
+                # Generar bucket_uri
+                bucket_uri = f"gs://{self.bucket_name}"
+
+                # Guardar bucket_name y bucket_uri en las variables de la tarea
+                self.setTaskVar("bucket_name", self.bucket_name)
+                self.setTaskVar("bucket_uri", bucket_uri)
+
+                return bucket_uri, message
+
+        except ComponentError as ce:
+            raise ce  # Re-lanzar errores específicos de componentes
+        except Exception as e:
+            raise ComponentError(f"CreateGCSBucket: Error durante la creación del bucket: {e}") from e
+
+    async def close(self):
+        """Cierra la conexión AsyncDB."""
+        try:
+            if self.bq:
+                await self.bq.close()
+                self._logger.info("CreateGCSBucket: AsyncDB cerrado exitosamente.")
+        except Exception as e:
+            self._logger.error(f"CreateGCSBucket: Error al cerrar AsyncDB: {e}")

flowtask/components/CreateReport/CreateReport.py

@@ -0,0 +1,228 @@
+"""
+CreateReport.
+
+Using a Jinja2 Template to crete a Report and, optionally, send via Email.
+
+
+        Example:
+
+        ```yaml
+        CreateReport:
+          template_file: echelon_program_overview_raw.html
+          create_pdf: true
+          masks:
+            '{today}':
+            - today
+            - mask: '%m/%d/%Y'
+            '{created}':
+            - today
+            - mask: '%Y-%m-%d %H:%M:%s'
+            '{firstdate}':
+            - date_diff
+            - value: today
+              diff: 8
+              mode: days
+              mask: '%b %d, %Y'
+            '{lastdate}':
+            - yesterday
+            - mask: '%b %d, %Y'
+          send:
+            via: email
+            list: echelon_program_overview
+            message:
+              subject: 'Echelon Kohl''s VIBA Report for: ({today})'
+            arguments:
+              today_report: '{today}'
+              generated_at: '{created}'
+              firstdate: '{firstdate}'
+              lastdate: '{lastdate}'
+        ```
+
+    """
+import asyncio
+import datetime
+from typing import Dict, List, Callable
+from pathlib import Path
+from navconfig.logging import logging
+from notify import Notify
+from notify.models import Actor
+from ..flow import FlowComponent
+from ...template import getTemplateHandler
+from ...exceptions import ComponentError, FileNotFound
+from ...conf import BASE_DIR, TASK_PATH
+from .charts import loadChart
+from ..support import DBSupport
+
+
+class CreateReport(FlowComponent, DBSupport):
+    """
+    CreateReport
+
+    Overview
+
+        The CreateReport class is a component for creating rich reports and sending them via the Notify service. It uses
+        template handling and chart creation to generate the content of the reports and sends them to a list of recipients.
+
+    .. table:: Properties
+    :widths: auto
+
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | Name             | Required | Description                                                                                      |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _data            |   Yes    | A dictionary containing the input data for the report.                                           |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _parser          |   Yes    | The template parser for generating the report content.                                           |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | _chartparser     |   Yes    | The template parser for generating charts.                                                       |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | template_file    |   Yes    | The file name of the template to use for the report.                                             |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | recipients       |   Yes    | A list of recipients to send the report to.                                                      |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | send             |   Yes    | A dictionary containing the sending options and configurations.                                  |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+        | message          |   No     | The message content for the report.                                                              |
+        +------------------+----------+-----------+--------------------------------------------------------------------------------------+
+
+        Returns:
+            The input data after sending the report.
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._data: Dict = {}
+        self._parser: Callable = None
+        self._chartparser: Callable = None
+        self.template_file: str = None
+        self.recipients = []
+        self.send: Dict = None
+        self.message: str = None
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        template_dir = Path(TASK_PATH, self._program, "templates")
+        echart_dir = Path(BASE_DIR, "DataIntegration", "dataintegration", "templates")
+        if self.previous:
+            self._data = self.input
+        try:
+            self._parser = getTemplateHandler(newdir=template_dir)
+            self._chartparser = getTemplateHandler(newdir=echart_dir)
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(
+                f"CreateReport: Unable to load Template Parser: {err}"
+            ) from err
+        template_file = template_dir.joinpath(self.template_file)
+        if not template_file.is_file():
+            raise FileNotFound(f"CreateReport: Missing Template File: {template_file}")
+        if hasattr(self, "masks"):
+            for key, value in self.send.items():
+                self.send[key] = self.mask_replacement(value)
+        # getting the mailing list:
+        try:
+            lst = self.send["list"]
+        except AttributeError:
+            lst = "owner"
+        sql = f"SELECT * FROM troc.get_mailing_list('{lst!s}')"
+        db = self.get_connection()
+        async with await db.connection() as conn:
+            try:
+                result, error = await conn.query(sql)
+                if error:
+                    raise ComponentError(
+                        f"CreateReport: Error on Recipients: {error!s}."
+                    )
+                for r in result:
+                    actor = Actor(**dict(r))
+                    self.recipients.append(actor)
+            except Exception as err:
+                logging.exception(err)
+        if not self.recipients:
+            raise ComponentError("CreateReport: Invalid Number of Recipients.")
+
+    def status_sent(self, recipient, message, result, *args, **kwargs):
+        print(f"Notification status {result!s} to {recipient.account!s}")
+        # logger:
+        logging.info(f"Notification status {result!s} to {recipient.account!s}")
+        # TODO: register sent on a logger database
+
+    async def send_message(self, recipients: List, data: Dict = None):
+        try:
+            emailService = Notify(self.send["via"], loop=self._loop)
+        except Exception as err:
+            raise ComponentError(f"Error loading Notify Service: {err!s}") from err
+        # add event for sent function
+        emailService.sent = self.status_sent
+        self.message = self.send["message"]
+        # create echart graph based on template
+        graph = {}
+        if hasattr(self, "echarts"):
+            for dataset, chart in self.echarts["charts"].items():
+                ds = self._data[dataset]
+                graph_name = chart["name"]
+                try:
+                    graphtype = chart["type"]
+                    del chart["type"]
+                except KeyError:
+                    graphtype = "bar"
+                try:
+                    title = chart["title"]
+                    del chart["title"]
+                except KeyError:
+                    title = dataset
+                chart = loadChart(graphtype, data=ds, title=title, **chart)
+                img = chart.image()
+                graph[graph_name] = img
+        # else:
+        emailTemplate = self._parser.get_template(self.template_file)
+        for user in recipients:
+            # TODO: add dot notation to Actor Model
+            email = user.account["address"]
+            args = {
+                "user": user,
+                "html": emailTemplate,
+                "email": email,
+                "created_at": datetime.datetime.now(),
+                **graph,
+            }
+            self.message["created_at"] = datetime.datetime.now()
+            other = {}
+            if "arguments" in self.send:
+                other = self.send["arguments"]
+            args = {**data, **args, **other}
+            content = emailTemplate.render(args)
+            try:
+                logging.debug(f"Sending Email to {user} via {email}")
+                # TODO: Send email via Worker or thread (async)
+                await emailService.send(
+                    recipient=user, template=None, message=content, **self.message
+                )
+            except Exception as err:
+                logging.exception(err)
+                raise ComponentError(f"Error Sending Email: {err}") from err
+            finally:
+                try:
+                    await emailService.close()
+                except Exception as err:
+                    logging.error(err)
+
+    def close(self):
+        pass
+
+    async def run(self):
+        try:
+            await self.send_message(self.recipients, self._data)
+            self._result = self.data
+            return self._result
+        except Exception as err:
+            logging.exception(err)
+            raise ComponentError(
+                f"CreateReport: Error sending Report: {err!s}"
+            ) from err

flowtask/components/CreateReport/__init__.py

@@ -0,0 +1,9 @@
+"""
+CreateReport.
+
+Component for creating Rich Reports and send via Notify.
+"""
+
+from .CreateReport import CreateReport
+
+__all__ = ["CreateReport"]

flowtask/components/CreateReport/charts/__init__.py

@@ -0,0 +1,15 @@
+"""Chart utilities using PyEcharts."""
+from typing import Any
+from .pie import pieChart
+from .bar import barChart
+
+__all__ = ["pieChart", "barChart"]
+
+
+def loadChart(charttype: str, title: str, data: Any, **kwargs):
+    if charttype == "bar":
+        return barChart(title, data, **kwargs)
+    elif charttype == "pie":
+        return pieChart(title, data, **kwargs)
+    else:
+        raise RuntimeError("CreateReport: Invalid Chart Type")

flowtask/components/CreateReport/charts/bar.py

@@ -0,0 +1,51 @@
+from typing import Any, Union, List, Dict
+import orjson
+from pyecharts.charts import Bar
+from borax.datasets.fetch import fetch
+import pyecharts.options as opts
+from .base import baseChart
+
+
+class barChart(baseChart):
+    rosetype: str = "radius"
+    formatter: str = "{c}"
+    axis_type: str = "category"
+
+    def __init__(self, title: str, data: Union[List, Dict], **kwargs):
+        fields = kwargs["fields"]
+        self._x = fields["x_axis"]
+        self._y = fields["y_axis"]
+        super(barChart, self).__init__(title, data, **kwargs)
+
+    def get_chart(self):
+        return Bar(init_opts=self.init_opts)
+
+    def set_chart(self, data: Any, title: str, **kwargs):
+        # x, y = Base.cast(o_data)
+        dst = self._encoder(data)
+        dst = orjson.loads(dst)
+        names = fetch(dst, self._x)
+        self.chart.add_xaxis(names)
+        for y in self._y:
+            values = fetch(dst, y)
+            self.chart.add_yaxis(y, values)
+        self.chart.set_global_opts(
+            **self.get_global_opts(title, **kwargs)
+        ).set_series_opts(**self.get_series_opts(**kwargs))
+
+    def get_basic_args(self, **kwargs):
+        return {}
+
+    def get_series_opts(self, **kwargs):
+        return {
+            "label_opts": opts.LabelOpts(
+                position="outside", formatter=self.formatter, is_show=True
+            )
+        }
+
+    def get_global_opts(self, title: str, **kwargs):
+        return {
+            "title_opts": opts.TitleOpts(title=title),
+            "legend_opts": opts.LegendOpts(is_show=True, pos_right="15%"),
+            "xaxis_opts": opts.AxisOpts(type_=self.axis_type),
+        }

flowtask/components/CreateReport/charts/base.py

@@ -0,0 +1,66 @@
+"""Base Class for all Chart types."""
+from abc import (
+    ABC,
+    abstractmethod,
+)
+from typing import Any, Union, List, Dict
+from pyecharts.globals import ThemeType
+import pyecharts.options as opts
+from pyecharts.render import make_snapshot
+from snapshot_selenium import snapshot as driver
+from asyncdb.utils.encoders import DefaultEncoder
+
+
+class baseChart(ABC):
+    theme: str = "LIGHT"
+
+    def __init__(self, title: str, data: Union[List, Dict], **kwargs):
+        self.formatter = None
+        if "theme" in kwargs:
+            self.theme = kwargs["theme"]
+            del kwargs["theme"]
+        try:
+            self._fields = kwargs["fields"]
+        except KeyError:
+            self._fields = ["name", "value"]
+        self._encoder = DefaultEncoder(sort_keys=False)
+        self._theme = getattr(ThemeType, self.theme)
+        # TODO: making more initializations
+        self.init_opts = opts.InitOpts(theme=self._theme, width="620px", height="420px")
+        self.chart = self.get_chart()
+        self.args = self.get_basic_args(**kwargs)
+        self.set_chart(data, title, **kwargs)
+
+    @abstractmethod
+    def get_chart(self):
+        pass
+
+    @abstractmethod
+    def set_chart(self, data: Any, title: str, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_basic_args(self, **kwargs):
+        pass
+
+    @abstractmethod
+    def get_global_opts(self, title: str, **kwargs):
+        pass
+
+    def get_series_opts(self, **kwargs):
+        return {
+            "label_opts": opts.LabelOpts(
+                position="outside", formatter=self.formatter, is_show=True
+            )
+        }
+
+    def chart(self):
+        return self.chart
+
+    def image(self):
+        """Returns the Base64 version of Image."""
+        img = None
+        make_snapshot(driver, self.chart.render(), "graph.base64")
+        with open("graph.base64", "r", encoding="utf-8") as f:
+            img = f.read()
+        return img