flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/events/events/exec.py

@@ -0,0 +1,124 @@
+import asyncio
+from datetime import datetime
+import traceback
+import socket
+from asyncdb import AsyncDB
+from ...conf import (
+    ENVIRONMENT,
+    TASK_EXEC_BACKEND,
+    TASK_EXEC_CREDENTIALS,
+    TASK_EVENT_TABLE,
+    INFLUX_DATABASE,
+    TASK_EXEC_TABLE,
+    USE_TASK_EVENT,
+)
+from ...utils.json import json_encoder
+from .abstract import AbstractEvent
+
+EVENT_HOST = socket.gethostbyname(socket.gethostname())
+
+
+class LogExecution(AbstractEvent):
+    """LogExecution.
+
+    Log the execution of a Task into a InfluxDB measurement bucket.
+    """
+
+    def get_influx(self):
+        return AsyncDB(
+            TASK_EXEC_BACKEND,
+            params=TASK_EXEC_CREDENTIALS,
+            loop=asyncio.get_event_loop(),
+        )
+
+    async def __call__(self, *args, **kwargs):
+        status = kwargs.pop("status", "event")
+        task = kwargs.pop("task", None)
+        cls = kwargs.pop("cls", None)
+        msg = kwargs.pop("message", str(cls))
+        if self.disable_notification is True:
+            return
+        if USE_TASK_EVENT is True:
+            try:
+                task_id = task.task_id
+                task_name = task.taskname
+                program = task.getProgram()
+            except (AttributeError, TypeError):
+                task_id = None
+                task_name = None
+                program = None
+            async with await self.get_influx().connection() as conn:
+                try:
+                    # saving the log into metric database:
+                    start_time = datetime.utcnow()
+                    data = {
+                        "measurement": TASK_EXEC_TABLE,
+                        "location": ENVIRONMENT,
+                        "timestamp": start_time,
+                        "fields": {"status": status},
+                        "tags": {
+                            "host": EVENT_HOST,
+                            "region": ENVIRONMENT,
+                            "start_time": start_time,
+                            "tenant": program,
+                            "task": task_name,
+                            "task_id": str(task_id),
+                            "message": msg,
+                            "traceback": str(traceback.format_exc()),
+                        },
+                    }
+                    await conn.write(data, bucket=INFLUX_DATABASE)
+                except Exception as e:
+                    self._logger.error(f"Flowtask: Error saving Task Execution: {e}")
+
+
+class SaveExecution(LogExecution):
+    async def __call__(self, *args, **kwargs):
+        status = kwargs.pop("status", "event")
+        task = kwargs.pop("task", None)
+        cls = kwargs.pop("error", None)
+        err = getattr(cls, "message", str(cls))
+        msg = kwargs.pop("message", err)
+        if self.disable_notification is True:
+            return
+        if USE_TASK_EVENT is True:
+            try:
+                stat = task.stats  # getting the stat object:
+            except AttributeError:
+                stats = None
+            if stat:
+                stats = json_encoder(stat.to_json())
+                start_time = stat.start_time
+                end_time = stat.finish_time
+                duration = stat.duration
+            else:
+                stats = {}
+                start_time = None
+                end_time = datetime.utcnow()
+                duration = None
+            async with await self.get_influx().connection() as conn:
+                try:
+                    data = {
+                        "measurement": TASK_EVENT_TABLE,
+                        "location": ENVIRONMENT,
+                        "timestamp": end_time,
+                        "fields": {"status": status},
+                        "tags": {
+                            "host": EVENT_HOST,
+                            "region": ENVIRONMENT,
+                            "stats": stats,
+                            "start_time": start_time,
+                            "finish_time": end_time,
+                            "duration": duration,
+                            "tenant": task.getProgram(),
+                            "task": task.taskname,
+                            "id": task.task_id,
+                            "traceback": traceback.format_exc(),
+                            "message": msg,
+                        },
+                    }
+                    await conn.write(data, bucket=INFLUX_DATABASE)
+                except Exception as e:
+                    self._logger.error(
+                        f"FlowTask: Error saving Task Execution: {e}"
+                    )

flowtask/events/events/file/__init__.py

@@ -0,0 +1,7 @@
+from .delete import FileDelete
+from .copy import FileCopy
+
+__all__ = (
+    "FileDelete",
+    "FileCopy",
+)

flowtask/events/events/file/base.py

@@ -0,0 +1,51 @@
+from abc import ABC
+import glob
+from pathlib import Path, PurePath
+from ..abstract import AbstractEvent
+
+
+class FileBase(AbstractEvent, ABC):
+    def __init__(self, *args, **kwargs):
+        self._filenames: list[PurePath] = []
+        super(FileBase, self).__init__(*args, **kwargs)
+
+    def start(self):
+        if hasattr(self, "directory"):
+            d = self.mask_replacement(
+                self.directory  # pylint: disable=E0203
+            )  # pylint: disable=access-member-before-definition
+            p = Path(d)  # pylint: disable=E0203
+            if p.exists() and p.is_dir():
+                self.directory = p
+            else:
+                self._logger.error(f"Path doesn't exists: {self.directory}")
+        if hasattr(self, "destination"):
+            d = self.mask_replacement(
+                self.destination  # pylint: disable=E0203
+            )  # pylint: disable=access-member-before-definition
+            p = Path(d)  # pylint: disable=E0203
+            if p.exists() and p.is_dir():
+                self.destination = p
+            else:
+                if hasattr(self, "create_destination"):
+                    self.destination.mkdir(parents=True, exist_ok=True)
+                else:
+                    self._logger.error(
+                        f"Destination Path doesn't exists: {self.directory}"
+                    )
+        if hasattr(self, "filename"):
+            if isinstance(self.filename, list):
+                for file in self.filename:
+                    fname = self.mask_replacement(file)
+                    self._filenames.append(self.directory.joinpath(fname))
+            elif isinstance(self.filename, PurePath):
+                self._filenames.append(self.filename)
+            elif isinstance(self.filename, str):
+                if "*" in self.filename:
+                    path = self.directory.joinpath(self.filename)
+                    listing = glob.glob(str(path))  # TODO using glob from pathlib
+                    for fname in listing:
+                        self._filenames.append(Path(fname))
+                else:
+                    fname = self.mask_replacement(self.filename)
+                    self._filenames.append(self.directory.joinpath(fname))

flowtask/events/events/file/copy.py

@@ -0,0 +1,23 @@
+import shutil
+from .base import FileBase
+
+
+class FileCopy(FileBase):
+    def __init__(self, *args, **kwargs):
+        self.move = kwargs.pop("move", False)
+        super(FileCopy, self).__init__(*args, **kwargs)
+
+    async def __call__(self, *args, **kwargs):
+        self.start()
+        for src in self._filenames:
+            dest = self.destination.joinpath(src.name)
+            if self.move:
+                shutil.move(src, dest)
+                # self._logger.info(
+                #     f"Moved file {src} to {dest}"
+                # )
+            else:
+                shutil.copy2(src, dest)
+                # self._logger.info(
+                #     f"Copied file {src} to {dest}"
+                # )

flowtask/events/events/file/delete.py

@@ -0,0 +1,16 @@
+from .base import FileBase
+
+
+class FileDelete(FileBase):
+    async def __call__(self, *args, **kwargs):
+        self.start()
+        for filename in self._filenames:
+            if filename.exists() and filename.is_file():
+                try:
+                    filename.unlink()
+                except Exception as e:
+                    self._logger.error(f"Error deleting {filename}: {e}")
+            else:
+                self._logger.warning(
+                    f"{filename} does not exist or is not a file."
+                )

flowtask/events/events/interfaces/__init__.py

@@ -0,0 +1,9 @@
+from .credentials import CredentialsInterface
+from .client import ClientInterface
+from .notifications import Notification
+
+__all__ = (
+    "ClientInterface",
+    "CredentialsInterface",
+    "Notification",
+)

flowtask/events/events/interfaces/client.py

@@ -0,0 +1,67 @@
+from abc import abstractmethod
+from collections.abc import Callable
+from typing import TypeVar
+from typing_extensions import ParamSpec
+from tqdm import tqdm
+from navconfig.logging import logging
+from .credentials import CredentialsInterface
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+class ClientInterface(CredentialsInterface):
+    def __init__(self, *args: P.args, **kwargs: P.kwargs) -> None:
+        super(ClientInterface, self).__init__(*args, **kwargs)
+        # host and port (if needed)
+        self.host: str = kwargs.pop("host", None)
+        if not self.host:
+            self.host = kwargs.pop("hostname", None)
+        self.port: int = kwargs.pop("port", None)
+        self.define_host()
+        self._connection: Callable = None
+        # progress bar
+        self._pb: Callable = None
+        # any other argument
+        self._clientargs = {}  # kwargs
+
+    def define_host(self):
+        try:
+            self.host = self.credentials["host"]
+        except KeyError:
+            self.host = self.host
+        try:
+            self.port = self.credentials["port"]
+        except KeyError:
+            self.port = self.port
+        # getting from environment:
+        self.host = self.get_env_value(self.host, default=self.host)
+        self.port = self.get_env_value(str(self.port), default=self.port)
+        if self.host:
+            logging.debug(f"<{__name__}>: HOST: {self.host}, PORT: {self.port}")
+
+    @abstractmethod
+    async def close(self, timeout: int = 5):
+        """close.
+        Closing the connection.
+        """
+
+    @abstractmethod
+    async def open(self, credentials: dict, **kwargs):
+        """open.
+        Starts (open) a connection to an external resource.
+        """
+
+    async def __aenter__(self) -> "ClientInterface":
+        await self.open(credentials=self.credentials, **self._clientargs)
+        return self
+
+    async def __aexit__(self, exc_type, exc_val, exc_tb) -> None:
+        # clean up anything you need to clean up
+        return await self.close(timeout=1)
+
+    def start_progress(self, total: int = 1):
+        self._pb = tqdm(total=total)
+
+    def close_progress(self):
+        self._pb.close()

flowtask/events/events/interfaces/credentials.py

@@ -0,0 +1,28 @@
+from abc import ABC
+from typing import TypeVar
+from typing_extensions import ParamSpec
+from ....interfaces.mask import MaskSupport
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+class CredentialsInterface(MaskSupport, ABC):
+    def __init__(self, *args: P.args, **kwargs: P.kwargs) -> None:
+        super(CredentialsInterface, self).__init__(*args, **kwargs)
+        self.credentials = self.set_credentials(kwargs.pop("credentials", {}))
+        if not self.credentials:
+            raise ValueError("Missing Credentials on Event Component")
+
+    def set_credentials(self, credentials: dict):
+        for key, default in credentials.items():
+            try:
+                # can process the credentials, extracted from environment or variables:
+                val = self.get_env_value(credentials[key], default=default)
+                if hasattr(self, "mask_replacement"):
+                    val = self.mask_replacement(val)
+                credentials[key] = val
+            except (TypeError, KeyError) as ex:
+                self._logger.error(f"{__name__}: Wrong or missing Credentias")
+                raise ValueError(f"{__name__}: Wrong or missing Credentias") from ex
+        return credentials

flowtask/events/events/interfaces/notifications.py

@@ -0,0 +1,58 @@
+from abc import ABC
+from notify import Notify
+from notify.providers.email import Email
+from notify.providers.slack import Slack
+from notify.providers.teams import Teams
+from notify.models import Actor, Chat, Channel, TeamsChannel
+from notify.conf import MS_TEAMS_DEFAULT_TEAMS_ID
+from flowtask.conf import (
+    EVENT_CHAT_ID,
+    EVENT_CHAT_BOT,
+    DEFAULT_RECIPIENT,
+    EMAIL_USERNAME,
+    EMAIL_PASSWORD,
+    EMAIL_PORT,
+    EMAIL_HOST,
+    SLACK_DEFAULT_CHANNEL,
+    SLACK_DEFAULT_CHANNEL_NAME,
+    MS_TEAMS_NAVIGATOR_CHANNEL,
+    MS_TEAMS_NAVIGATOR_CHANNEL_ID,
+)
+
+
+class Notification(ABC):
+    def get_notify(self, _type, **kwargs):
+        if _type == "telegram":
+            # defining the Default chat object:
+            recipient = Chat(**{"chat_id": EVENT_CHAT_ID, "chat_name": "Navigator"})
+            # send notifications to Telegram bot
+            args = {"bot_token": EVENT_CHAT_BOT, **kwargs}
+            ntf = Notify("telegram", **args)
+        elif _type == "slack":
+            recipient = Channel(
+                channel_id=SLACK_DEFAULT_CHANNEL,
+                channel_name=SLACK_DEFAULT_CHANNEL_NAME,
+            )
+            ntf = Slack()
+        elif _type == "email":
+            account = {
+                "host": EMAIL_HOST,
+                "port": EMAIL_PORT,
+                "username": EMAIL_USERNAME,
+                "password": EMAIL_PASSWORD,
+                **kwargs,
+            }
+            recipient = Actor(**DEFAULT_RECIPIENT)
+            ntf = Email(debug=True, **account)
+        elif _type == "teams":
+            ntf = Teams(as_user=True)
+            recipient = TeamsChannel(
+                name=MS_TEAMS_NAVIGATOR_CHANNEL,
+                team_id=MS_TEAMS_DEFAULT_TEAMS_ID,
+                channel_id=MS_TEAMS_NAVIGATOR_CHANNEL_ID,
+            )
+        else:
+            # Any other Notify Provider:
+            recipient = Actor(**DEFAULT_RECIPIENT)
+            ntf = Notify(_type, **kwargs)
+        return [ntf, recipient]

flowtask/events/events/jira.py

@@ -0,0 +1,122 @@
+import codecs
+import traceback
+from jira import JIRA
+import orjson
+from ...utils.json import json_encoder
+from .abstract import AbstractEvent
+from ...conf import (
+    JIRA_API_TOKEN,
+    JIRA_USERNAME,
+    JIRA_INSTANCE,
+    JIRA_PROJECT
+)
+
+
+class Jira(AbstractEvent):
+    """Jira.
+
+    Jira Event to create a new Ticket on Error/Exception.
+    """
+    def __init__(self, *args, **kwargs):
+        super(Jira, self).__init__(*args, **kwargs)
+        self.program = kwargs.pop("program", None)
+        self.task = kwargs.pop("task", None)
+        self.issue_type = kwargs.pop("issue_type", 'Bug')
+        self._status = kwargs.pop("status", "event")
+        self._summary = kwargs.get('summary', 'Error on: ')
+        self._assignee = kwargs.get('assignee', {})
+        # Initialize Jira Connection
+        try:
+            self.jira = JIRA(
+                server=JIRA_INSTANCE,
+                basic_auth=(JIRA_USERNAME, JIRA_API_TOKEN)
+            )
+        except Exception as err:
+            self._logger.error(
+                f"Cannot Connect to Jira {err}"
+            )
+            raise
+
+    async def __call__(self, *args, **kwargs):
+        project_id = kwargs.pop('project_id', JIRA_PROJECT)
+        _message = kwargs.get('message', '')
+        task = kwargs.pop("task", None)
+        status = kwargs.pop("status", "done")
+        program = task.getProgram()
+        task_name = f"{program}.{task.taskname}"
+        error = kwargs.get('error', None)
+        task_id = task.task_id
+        if status not in ("error", "failed", "exception", "task error"):
+            return True
+        # Operar el error
+        try:
+            stat = task.stats  # getting the stat object:
+            stats = stat.to_json()
+        except AttributeError:
+            stats = []
+        description = {}
+        if stats:
+            # iterate over task stats:
+            for stat, value in stats["steps"].items():
+                description[stat] = value
+        # Create the Jira Ticket:
+        desc = orjson.dumps(description, option=orjson.OPT_INDENT_2).decode()
+        codereview = codecs.decode(json_encoder(stats['Review']), 'unicode_escape').replace("**", "*")
+        desc = f"""
+        {{code:json}}
+        {desc}
+        {{code}}\n\n
+        """
+        # Capture the stack trace
+        stack_trace = traceback.format_exc()
+        if stack_trace:
+            desc = f"""
+            {desc}
+
+            h2. *Error*:\n\n
+            {{panel:bgColor=#ffebe6}}\n
+            {error}
+            \n{{panel}}\n\n
+
+            h2. Stack Trace:\n\n
+            {{code:python}}
+            {stack_trace}
+            {{code}}
+            """
+        args = {
+            'project': project_id,
+            'summary': f"{self._summary} {task_name}",
+            'description': (
+                f"{{color:#FF5630}}[ STATUS: {status.upper()} ]{{color}} \n\n"
+                f"{{panel:bgColor=#deebff}}\n*Task*: {task_name}\n{{panel}}\n\n"
+                f"{{panel:bgColor=#deebff}}\n*Task_id*: {task_id}\n{{panel}}\n\n"
+                f"{{panel:bgColor=#ffebe6}}\n*Message*: {_message}\n{{panel}}\n\n"
+                f"h2. *Task Stats*:\n\n"
+                f"{desc}"
+                f"\n\n----\n\n{{panel:bgColor=#eae6ff}}\nh2. *Code Review*\n\n{codereview}\n\n{{panel}}"
+            ),
+            'issuetype': {
+                'name': self.issue_type
+            }
+        }
+        try:
+            new_issue = self.jira.create_issue(**args)
+            self._logger.info(
+                f"Jira Issue Created: {new_issue}"
+            )
+            if self._assignee:
+                try:
+                    new_issue.update(
+                        assignee=self._assignee
+                    )
+                except Exception as err:
+                    print('ERROR > ', err)
+                    self._logger.warning(
+                        f"Error Assigning Jira Issue to {self._assignee}: {err}"
+                    )
+            return new_issue
+        except Exception as err:
+            self._logger.error(
+                f"Cannot Create Jira Issue {err}"
+            )
+            raise

flowtask/events/events/log.py

@@ -0,0 +1,26 @@
+from navconfig import DEBUG
+from navconfig.logging import logging
+from .abstract import AbstractEvent
+
+
+class LogEvent(AbstractEvent):
+    def __init__(self, *args, **kwargs):
+        super(LogEvent, self).__init__(*args, **kwargs)
+        self.use_notice: bool = kwargs.pop("use_notice", False)
+        self._logger = logging.getLogger(
+            "FlowTask.LogEvent"
+        )
+
+    async def __call__(self, *args, **kwargs):
+        msg = kwargs.pop("message", None)
+        cls = kwargs.pop("cls", None)
+        if not msg:
+            msg = getattr(cls, "message", str(cls))
+        if DEBUG:
+            self.echo(msg, level="WARN")
+            self._logger.debug(msg)
+        else:
+            if self.use_notice is True:
+                self._logger.notice(msg)
+            else:
+                self._logger.info(msg)

flowtask/events/events/logerr.py

@@ -0,0 +1,52 @@
+import socket
+from datetime import datetime, timezone
+from redis import asyncio as aioredis
+from navconfig.logging import logging
+from ...utils.json import json_encoder
+from .abstract import AbstractEvent
+from ...conf import ENVIRONMENT, PUBSUB_REDIS, ERROR_CHANNEL
+
+
+EVENT_HOST = socket.gethostbyname(socket.gethostname())
+
+
+class LogError(AbstractEvent):
+    def __init__(self, *args, **kwargs):
+        super(LogError, self).__init__(*args, **kwargs)
+        self._logger = logging.getLogger("FlowTask.LogError")
+
+    async def __call__(self, *args, **kwargs):
+        status = kwargs.pop("status", "event")
+        task = kwargs.pop("task", None)
+        program = task.getProgram()
+        task_name = task.taskname
+        task_id = task.task_id
+        msg = kwargs.pop("message", None)
+        cls = kwargs.pop("cls", None)
+        if not msg:
+            msg = getattr(cls, "message", str(cls))
+        redis = await aioredis.from_url(
+            PUBSUB_REDIS, encoding="utf-8", decode_responses=True
+        )
+        msg = {
+            "task": f"{program}.{task_name}",
+            "task_id": task_id,
+            "type": "error",
+            "status": status,
+            "environment": ENVIRONMENT,
+            "host": EVENT_HOST,
+            "end_time": datetime.now(timezone.utc),
+        }
+        message = json_encoder(msg)
+        try:
+            await redis.publish(ERROR_CHANNEL, message)
+        except Exception as e:
+            self._logger.warning(
+                f"Redis Publisher Error: {e}"
+            )
+        finally:
+            await redis.close()
+            try:
+                await redis.connection_pool.disconnect()
+            except Exception:
+                pass

flowtask/events/events/notify.py

@@ -0,0 +1,59 @@
+from ...interfaces.notification import Notification
+from ...exceptions import ActionError
+from .abstract import AbstractEvent
+
+
+class Notify(Notification, AbstractEvent):
+    async def __call__(self, *args, **kwargs):
+        default_message = self._kwargs.pop("message", None)
+        message = kwargs.pop("message", default_message)
+        # status = kwargs.pop('status', 'event')
+        task = kwargs.pop("task", None)
+        provider, recipient = self.get_notify(**self._kwargs)
+        # Mask transform of message
+        for key, value in self.message.items():
+            self.message[key] = self.mask_replacement(value)
+        ## TASK:
+        if self.provider == "email":
+            args = self.get_message(message)
+            if hasattr(self, "with_attachment") and self.with_attachment is True:
+                # check if result are files (build attachments)
+                _vars = task.get_variables()
+                if "FILENAMES" in _vars.keys():
+                    result = _vars["FILENAMES"]
+                elif "FILENAME" in _vars.keys():
+                    result = _vars["FILENAME"]
+                else:
+                    result = kwargs.pop("result", [])
+                self.list_attachment = self.get_attachment_files(result)
+        elif self.provider == "telegram":
+            args = self.get_message(message)
+            if hasattr(self, "with_attachment") and self.with_attachment is True:
+                _vars = task.get_variables()
+                if "FILENAMES" in _vars.keys():
+                    result = _vars["FILENAMES"]
+                elif "FILENAME" in _vars.keys():
+                    result = _vars["FILENAME"]
+                else:
+                    result = kwargs.pop("result", [])
+                self.list_attachment = self.get_attachment_files(result)
+                async with provider as notify:
+                    result = await notify.send_document(
+                        document=self.list_attachment[0],
+                        caption=message,
+                        disable_notification=True,
+                    )
+                    return result
+        else:
+            args = self.get_message(message)
+        if self._template:
+            args["template"] = self._template
+        try:
+            async with provider as notify:
+                args["recipient"] = recipient
+                if self.list_attachment:
+                    args["attachments"] = self.list_attachment
+                result = await notify.send(**args)
+                self._logger.debug(f"Notification Status: {result}")
+        except Exception as err:
+            raise ActionError(f"Error Creating Notification App: {err}") from err