flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/events/events/notify_event.py

@@ -0,0 +1,160 @@
+# Logging:
+from navconfig.logging import logging
+
+## Notify System
+from notify import Notify
+from notify.providers.email import Email
+from notify.providers.slack import Slack
+from notify.providers.teams import Teams
+from notify.models import (
+    Actor,
+    Chat,
+    Channel,
+    TeamsCard,
+    TeamsChannel
+)
+from ...conf import (
+    SEND_NOTIFICATIONS,
+    EVENT_CHAT_ID,
+    EVENT_CHAT_BOT,
+    NOTIFY_ON_ERROR,
+    NOTIFY_ON_SUCCESS,
+    NOTIFY_ON_FAILURE,
+    NOTIFY_ON_WARNING,
+    DEFAULT_RECIPIENT,
+    EMAIL_USERNAME,
+    EMAIL_PASSWORD,
+    EMAIL_PORT,
+    EMAIL_HOST,
+    ENVIRONMENT,
+    SLACK_DEFAULT_CHANNEL,
+    SLACK_DEFAULT_CHANNEL_NAME,
+    MS_TEAMS_DEFAULT_TEAMS_ID,
+    MS_TEAMS_DEFAULT_CHANNEL_ID,
+    MS_TEAMS_DEFAULT_CHANNEL_NAME,
+    SHOW_VERSION
+)
+from ...utils.functions import check_empty
+from .abstract import AbstractEvent
+from ...version import __version__
+
+
+class NotifyEvent(AbstractEvent):
+    """Using Notify to send notifications for task Execution."""
+
+    def __init__(self, *args, event: str = "done", **kwargs):
+        super(NotifyEvent, self).__init__(*args, **kwargs)
+        self._logger = logging.getLogger("FlowTask.Notify")
+        self._event_ = event
+        if event == "done":
+            self.event = NOTIFY_ON_SUCCESS
+        elif event == "warning":
+            self.event = NOTIFY_ON_WARNING
+        elif event == "error":
+            self.event = NOTIFY_ON_ERROR
+        elif event == "exception":
+            self.event = NOTIFY_ON_FAILURE
+        else:
+            self.event = NOTIFY_ON_SUCCESS
+
+    def getNotify(self, notify, **kwargs):
+        if notify == "telegram":
+            # defining the Default chat object:
+            recipient = Chat(**{"chat_id": EVENT_CHAT_ID, "chat_name": "Navigator"})
+            # send notifications to Telegram bot
+            args = {"bot_token": EVENT_CHAT_BOT, **kwargs}
+            ntf = Notify("telegram", **args)
+        elif notify == "slack":
+            recipient = Channel(
+                channel_id=SLACK_DEFAULT_CHANNEL,
+                channel_name=SLACK_DEFAULT_CHANNEL_NAME,
+            )
+            ntf = Slack()
+        elif notify == "email":
+            account = {
+                "host": EMAIL_HOST,
+                "port": EMAIL_PORT,
+                "username": EMAIL_USERNAME,
+                "password": EMAIL_PASSWORD,
+                **kwargs,
+            }
+            recipient = Actor(**DEFAULT_RECIPIENT)
+            ntf = Email(debug=True, **account)
+        elif notify == 'teams':
+            team_id = kwargs.pop("team_id", MS_TEAMS_DEFAULT_TEAMS_ID)
+            recipient = TeamsChannel(
+                name=MS_TEAMS_DEFAULT_CHANNEL_NAME,
+                team_id=team_id,
+                channel_id=MS_TEAMS_DEFAULT_CHANNEL_ID
+            )
+            ntf = Teams(
+                as_user=True,
+                team_id=team_id,
+            )
+        else:
+            # Any other Notify Provider:
+            recipient = Actor(**DEFAULT_RECIPIENT)
+            ntf = Notify(notify, **kwargs)
+        return [ntf, recipient]
+
+    async def __call__(self, *args, **kwargs):
+        if SEND_NOTIFICATIONS is False:
+            return
+        task = kwargs.pop("task", None)
+        result = kwargs.pop("result", None)
+        message = kwargs.pop("message", None)
+        trace = kwargs.pop("stacktrace", None)
+        error = kwargs.pop("error", None)
+        program = task.getProgram()
+        task_name = task.taskname
+        component = kwargs.pop("component", None)
+        if error is not None:
+            self.event = NOTIFY_ON_ERROR
+            if program and task and component:
+                message = f"🛑 ::{ENVIRONMENT} -  Task {program}.{task_name}, Error on {component}: {error!s}"
+            elif program and task:
+                message = (
+                    f"🛑 ::{ENVIRONMENT} -  Error on {program}.{task_name}: {error!s}"
+                )
+            elif trace is not None:
+                self.event = NOTIFY_ON_FAILURE
+                message = f"🛑 ::{ENVIRONMENT} - {program}.{task_name}: {error!s}"
+            else:
+                message = f"🛑 ::{ENVIRONMENT} - {program}.{task_name}: {error!s}"
+        elif self._event_ == "warning":
+            message = f" ⚠️ :: {ENVIRONMENT} - *{program}.{task_name}*: Warning {component}->{str(message)!s}"
+        else:
+            if message is not None and result is not None:
+                # success event:
+                self.event = NOTIFY_ON_SUCCESS
+                message = f" ✅ :: {ENVIRONMENT} - {program}.{task_name}: {message!s}"
+            elif not check_empty(result):
+                message = f" ✅ :: {ENVIRONMENT} - {program}.{task_name}: {message!s}"
+            else:
+                message = f" ⚠️ :: {ENVIRONMENT} - {program}.{task_name}: {message!s}, Empty Result."
+
+        if SHOW_VERSION:
+            message = f"{message} - Version: {__version__}"
+        ntf, recipients = self.getNotify(self.event, **kwargs)
+        args = {"recipient": [recipients], "message": message}
+        if self.event == 'teams':
+            channel = recipients
+            msg = TeamsCard(
+                text=str(message),
+                summary=f"Task Summary: {program}.{task_name}",
+                title=f"Task {program}.{task_name}",
+            )
+            async with ntf as conn:
+                return await conn.send(
+                    recipient=channel,
+                    message=msg
+                )
+        elif ntf.provider_type == "email":
+            args["subject"] = message
+        elif ntf.provider == "telegram":
+            args["disable_notification"] = True
+        else:
+            args["subject"] = message
+        async with ntf as t:
+            result = await t.send(**args)
+        return result

flowtask/events/events/publish.py

@@ -0,0 +1,54 @@
+import socket
+from datetime import datetime, timezone
+from redis import asyncio as aioredis
+from navconfig.logging import logging
+from ...utils.json import json_encoder
+from ...conf import (
+    ENVIRONMENT,
+    PUBSUB_REDIS
+)
+from .abstract import AbstractEvent
+
+EVENT_HOST = socket.gethostbyname(socket.gethostname())
+
+
+class PublishEvent(AbstractEvent):
+    async def __call__(self, *args, **kwargs):
+        status = kwargs.pop("status", "event")
+        task = kwargs.pop("task", None)
+        program = task.getProgram()
+        task_name = task.taskname.replace(
+            ".", ":"
+        )  # Convert dots to colons for Redis channel name
+        channel_name = f"{program}:{task_name}"
+        task_id = task.task_id
+        redis = await aioredis.from_url(
+            PUBSUB_REDIS,
+            encoding="utf-8",
+            decode_responses=True
+        )
+        try:
+            stat = task.stats  # getting the stat object:
+            stats = json_encoder(stat.to_json())
+        except AttributeError:
+            stats = None
+        msg = {
+            "task": f"{program}.{task_name}",
+            "task_id": task_id,
+            "status": status,
+            "environment": ENVIRONMENT,
+            "host": EVENT_HOST,
+            "stats": stats,
+            "end_time": datetime.now(timezone.utc),
+        }
+        message = json_encoder(msg)
+        try:
+            await redis.publish(channel_name, message)
+        except Exception as e:
+            logging.warning(f"Event Publisher Error: {e}")
+        finally:
+            await redis.close()
+            try:
+                await redis.connection_pool.disconnect()
+            except Exception:
+                pass

flowtask/events/events/sendfile.py

@@ -0,0 +1,104 @@
+from typing import Union
+from collections.abc import Callable, Iterable
+import asyncio
+from pathlib import Path
+from notify.models import Actor
+from notify import Notify
+from ...exceptions import FileNotFound, ActionError
+from .interfaces import ClientInterface
+from .abstract import AbstractEvent
+
+
+def expand_path(filename: Union[str, Path]) -> Iterable[Path]:
+    if isinstance(filename, str):
+        p = Path(filename)
+    else:
+        p = filename
+    return list(Path(p.parent).expanduser().glob(p.name))
+
+
+class SendFile(ClientInterface, AbstractEvent):
+    def __init__(self, *args, **kwargs):
+        self.list_attachment: list = []
+        self.notify: Callable = None
+        super(SendFile, self).__init__(*args, **kwargs)
+
+    async def __call__(self, *args, **kwargs):
+        # determine the recipients:
+        # TODO: add support for mailing lists
+        try:
+            self._recipients = [Actor(**user) for user in self.recipients]
+        except Exception as err:
+            raise RuntimeError(f"Error formatting Recipients: {err}") from err
+        if not self._recipients:
+            raise RuntimeError("SendNotify: Invalid Number of Recipients.")
+
+        # File Attachment:
+        # TODO: multiple attachments
+        if hasattr(self, "directory"):
+            d = self.mask_replacement(
+                self.directory  # pylint: disable=access-member-before-definition
+            )
+            p = Path(d)  # pylint: disable=E0203
+            if p.exists() and p.is_dir():
+                self.directory = p
+            else:
+                self._logger.error(f"Path doesn't exists: {self.directory}")
+        else:
+            self.directory = None
+
+        if hasattr(self, "filename"):
+            file = self.mask_replacement(self.filename)
+            files = []
+            if self.directory:
+                fs = self.directory.joinpath(file)
+                files = expand_path(fs)
+            else:
+                files = expand_path(file)
+            for file in files:
+                if file.exists():
+                    self.list_attachment.append(file)
+                else:
+                    raise FileNotFound(f"File doesn't exists: {file}")
+
+        # Mask transform of message
+        for key, value in self.message.items():
+            self.message[key] = self.mask_replacement(value)
+
+        try:
+            await self.open()
+            async with self.notify as mail:
+                try:
+                    result = await mail.send(
+                        recipient=self._recipients,
+                        attachments=self.list_attachment,
+                        **self.message,
+                    )
+                    self._logger.debug(f"Notification Status: {result}")
+                except Exception as err:
+                    raise ActionError(f"SendNotify Error: {err}") from err
+                return None
+        finally:
+            await self.close()
+
+    async def close(self):
+        """close.
+        Closing the connection.
+        """
+        if self.notify:
+            try:
+                await self.notify.close()
+            except Exception as err:
+                print(err)
+
+    async def open(self):
+        """open.
+        Starts (open) a connection to an external resource.
+        """
+        try:
+            self.notify = Notify(
+                "email", loop=asyncio.get_event_loop(), **self.credentials
+            )
+        except Exception as err:
+            raise ActionError(f"Error Creating Notification App: {err}") from err
+        return self

flowtask/events/events/task.py

@@ -0,0 +1,97 @@
+import asyncio
+import pandas as pd
+from navconfig import DEBUG
+from navconfig.logging import logging
+from .abstract import AbstractEvent
+from ...exceptions import (
+    NotSupported,
+    TaskNotFound,
+    TaskFailed,
+    FileError,
+    FileNotFound,
+    DataNotFound,
+)
+
+
+class RunTask(AbstractEvent):
+    def __init__(self, *args, **kwargs):
+        super(RunTask, self).__init__(*args, **kwargs)
+        self.program = kwargs.pop("program", None)
+        self.task = kwargs.pop("task", None)
+
+    def run_task_in_thread(self, task_coroutine):
+        loop = asyncio.new_event_loop()  # Create a new event loop for the thread
+        asyncio.set_event_loop(loop)
+        loop.run_until_complete(task_coroutine)
+        loop.close()
+
+    async def task_execution(self):
+        # avoid circular import
+        from flowtask.tasks.task import Task  # noqa
+
+        result = {}
+        task = Task(
+            task=self.task,
+            program=self.program,
+            loop=asyncio.get_event_loop(),
+            enable_stat=False,
+            ignore_results=True,
+            debug=DEBUG,
+        )
+        try:
+            state = await task.start()
+            if not state:
+                logging.warning(
+                    f"Task {self.program}.{self.task} return False on Start Time."
+                )
+        except Exception as err:
+            logging.error(err)
+            raise
+        try:
+            state = await task.run()
+            try:
+                result["stats"] = task.stats.stats
+            except Exception as err:
+                result["stats"] = None
+                result["error"] = err
+            ### gettting the result of Task execution
+            if isinstance(state, pd.DataFrame):
+                numrows = len(state.index)
+                columns = list(state.columns)
+                num_cols = len(columns)
+                state = {
+                    "type": "Dataframe",
+                    "numrows": numrows,
+                    "columns": columns,
+                    "num_cols": num_cols,
+                }
+            else:
+                state = f"{state!r}"
+            result["result"] = state
+            await task.close()
+            return result
+        except DataNotFound as err:
+            raise
+        except NotSupported as err:
+            raise
+        except TaskNotFound as err:
+            raise TaskNotFound(f"Task: {self.task}: {err}") from err
+        except TaskFailed as err:
+            raise TaskFailed(f"Task {self.task} failed: {err}") from err
+        except FileNotFound:
+            raise
+        except FileError as err:
+            raise FileError(f"Task {self.task}, File Not Found error: {err}") from err
+        except Exception as err:
+            raise TaskFailed(f"Error: Task {self.task} failed: {err}") from err
+
+    async def __call__(self, *args, **kwargs):
+        self._logger.debug(f":: Running Task: {self.program}.{self.task}")
+        return await self.task_execution()
+        # task_coroutine = self.task_execution()
+        # task_thread = threading.Thread(
+        #     target=self.run_task_in_thread,
+        #     args=(task_coroutine,)
+        # )
+        # task_thread.start()
+        # task_thread.join()  # Wait for the thread to finish

flowtask/events/events/teams.py

@@ -0,0 +1,98 @@
+import codecs
+import orjson
+from notify.models import TeamsCard, TeamsChannel
+from notify.providers.teams import Teams
+from notify.conf import MS_TEAMS_DEFAULT_TEAMS_ID, MS_TEAMS_DEFAULT_CHANNEL_ID
+from ...utils.json import json_encoder
+from ...interfaces.env import EnvSupport
+from .abstract import AbstractEvent
+
+
+class TeamsMessage(AbstractEvent):
+    def __init__(self, *args, **kwargs):
+        super(TeamsMessage, self).__init__(*args, **kwargs)
+        self.channel = kwargs.get('channel', MS_TEAMS_DEFAULT_CHANNEL_ID)
+        self.team_id = kwargs.get('team_id', MS_TEAMS_DEFAULT_TEAMS_ID)
+        self.team_id = self.get_env_value(self.team_id, default=self.team_id)
+        self.channel = self.get_env_value(self.channel, default=self.channel)
+        self.channel_name = kwargs.get('channel_name', 'General')
+        self.channel_name = self.get_env_value(self.channel_name, default=self.channel_name)
+        self._text = self.mask_replacement(kwargs.pop("text", ''))
+        self._title = self.mask_replacement(kwargs.pop("title", 'Task Info:'))
+
+    async def __call__(self, *args, **kwargs):
+        tm = Teams(
+            as_user=True,
+            team_id=self.team_id,
+        )
+        channel = TeamsChannel(
+            name=self.channel_name,
+            team_id=self.team_id,
+            channel_id=self.channel,
+        )
+        status = kwargs.pop("status", "done")
+        task = kwargs.pop("task", None)
+        program = task.getProgram()
+        task_name = f"{program}.{task.taskname}"
+        task_id = task.task_id
+        message = kwargs.pop("message", f"Task Completed {task_name}, {task_id}")
+        message = self.mask_replacement(message)
+        summary = kwargs.pop("summary", f"Task Summary: {task_name}")
+        summary = self.mask_replacement(summary)
+        try:
+            stat = task.stats  # getting the stat object:
+            stats = stat.to_json()
+        except AttributeError:
+            stats = []
+        if status == "done":
+            icon = "✅"
+        elif status in ("error", "failed", "exception", "task error"):
+            icon = "🛑"
+        elif status in (
+            "warning",
+            "file_not_found",
+            "not_found",
+            "data_not_found",
+            "done_warning",
+        ):
+            icon = "⚠️"
+        elif status in ("empty_file"):
+            icon = "📄"
+        else:
+            icon = "✅"
+        txt = f"{icon} {message}"
+        if self.type == "card":
+            msg = TeamsCard(
+                title=f"Task {task_name} uid:{task_id}",
+                text=txt,
+                summary=summary,
+            )
+            if hasattr(self, "with_stats"):
+                # iterate over task stats:
+                for stat, value in stats["steps"].items():
+                    section = msg.addSection(activityTitle=stat, text=stat)
+                    stats_json = orjson.dumps(value, option=orjson.OPT_INDENT_2 | orjson.OPT_APPEND_NEWLINE).decode()
+                    section.addFacts(
+                        facts=[{"name": stat, "value": stats_json}]
+                    )
+            if hasattr(self, 'with_codereview'):
+                # Add Code Review
+                section = msg.addSection(activityTitle="Code Review", text="Code Review")
+                codereview = codecs.decode(json_encoder(stats['Review']), 'unicode_escape')
+                section.addFacts(
+                    facts=[{"name": "Code Review", "value": codereview}]
+                )
+            else:
+                # only one single block of text:
+                section = msg.addSection(
+                    activityTitle=self._title,
+                    text=self._text,
+                )
+                section.addFacts(
+                    facts=[{"name": "Info", "value": self._text}]
+                )
+        async with tm as conn:
+            return await conn.send(
+                recipient=channel,
+                message=msg
+            )

flowtask/events/events/webhook.py

@@ -0,0 +1,58 @@
+import aiohttp
+from ...utils.json import json_encoder
+from .abstract import AbstractEvent
+
+
+class WebHook(AbstractEvent):
+    def __init__(self, *args, **kwargs):
+        self.url = kwargs.pop("url", None)
+        super(WebHook, self).__init__(*args, **kwargs)
+
+    async def __call__(self, *args, **kwargs):
+        if not self.url:
+            raise ValueError("URL was not provided for WebHook event action.")
+        status = kwargs.pop("status", "event")
+        task = kwargs.pop("task", None)
+        program = task.getProgram()
+        task_name = f"{program}.{task.taskname}"
+        task_id = task.task_id
+        try:
+            stat = task.stats  # getting the stat object:
+            stats = json_encoder(stat.to_json())
+        except AttributeError:
+            stats = None
+
+        # Extract optional authentication parameters
+        auth_type = self._kwargs.get("auth_type", None)
+        auth_value = self._kwargs.get("auth_value", None)
+
+        headers = {}
+        if auth_type and auth_value:
+            if auth_type.lower() == "basic":
+                headers["Authorization"] = f"Basic {auth_value}"
+            elif auth_type.lower() == "bearer":
+                headers["Authorization"] = f"Bearer {auth_value}"
+            else:
+                raise ValueError(f"Unsupported authentication type: {auth_type}")
+
+        # Prepare the payload. You can customize this based on your needs.
+        payload = {
+            "task": task_name,
+            "id": str(task_id),
+            "status": status,
+            "message": kwargs.get("message", "Task Completed."),
+            "stats": stats,
+        }
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                self.url, json=payload, headers=headers
+            ) as response:
+                if response.status != 200:
+                    # Handle non-200 responses if necessary
+                    response = await response.text()
+                    self._logger.warning(
+                        f"Error on WebHook response: {response}, status: {response.status}"
+                    )
+                else:
+                    # Handle successful webhook call if necessary
+                    return