flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/hooks/actions/sampledata.py

@@ -0,0 +1,37 @@
+from .abstract import AbstractAction
+
+class ProcessData(AbstractAction):
+    """ProcessData.
+    Process data received in a webhook event.
+    """
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.params = kwargs
+
+    async def open(self):
+        pass
+
+    async def close(self):
+        pass
+
+    async def run(self, hook, *args, **kwargs):
+        """
+        Sample Action for Processing uploaded files.
+        """
+        uploaded_files = kwargs.get('uploaded_files', [])
+        form_data = kwargs.get('form_data', {})
+        notification = kwargs.get('notification', None)
+        if notification:
+            self._logger.notice(f"Notification: {notification}")
+            self._logger.info(
+                f"Processing notification from channel '{notification.channel}': {notification.payload}"
+            )
+        # Process each uploaded file
+        results = []
+        for file_path in uploaded_files:
+            # Open and process the file
+            with open(file_path, 'r') as f:
+                content = f.read()
+                # Do something with the content
+                results.append({'file': file_path, 'content_length': len(content)})
+        return {'results': results, 'form_data': form_data}

flowtask/hooks/actions/sensor.py

@@ -0,0 +1,23 @@
+from .abstract import AbstractAction
+from ...utils import cPrint
+
+class ProcessSensorData(AbstractAction):
+    """ProcessSensorData.
+
+    WIP example of to process Sensor data from MQTTT service.
+    """
+
+    async def open(self):
+        cPrint('Opening Action on Process.')
+
+    async def run(self, hook, *args, **kwargs):
+        print(
+            f"Running action from hook {hook} with arguments:",
+            args,
+            self._args,
+            kwargs,
+            self._kwargs
+        )
+
+    async def close(self):
+        cPrint("Closing Action on Process.")

flowtask/hooks/actions/task.py

@@ -0,0 +1,9 @@
+from .abstract import AbstractAction
+from ...interfaces.task import TaskSupport
+
+
+class Task(TaskSupport, AbstractAction):
+    """Task.
+
+    Calling an FlowTask Task.
+    """

flowtask/hooks/actions/ticket.py

@@ -0,0 +1,37 @@
+from abc import abstractmethod
+from collections.abc import Callable
+from .abstract import AbstractAction
+
+
+class AbstractTicket(AbstractAction):
+    """AbstractTicket.
+
+    Managing Ticket system using Actions.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(AbstractTicket, self).__init__(*args, **kwargs)
+        self.service: Callable = None
+        try:
+            self._action_ = self._kwargs["action"]
+            del self._kwargs["action"]
+        except KeyError:
+            self._action_ = "create"
+
+    @abstractmethod
+    async def create(self, *args, **kwargs):
+        """create.
+
+        Create a new Ticket.
+        """
+
+    async def run(self, hook, *args, **kwargs):
+        if not self.service:
+            await self.open()
+        try:
+            if self._action_ == "create":
+                params = {**self._kwargs, **kwargs}
+                return await self.create(*self._args, **params)
+        except Exception as exc:
+            self._logger.error(f"Error creating new Ticket: {exc}")
+            raise

flowtask/hooks/actions/zammad.py

@@ -0,0 +1,55 @@
+from ...conf import (
+    ZAMMAD_INSTANCE,
+    ZAMMAD_TOKEN,
+    ZAMMAD_DEFAULT_CUSTOMER,
+    ZAMMAD_DEFAULT_GROUP,
+)
+from ...exceptions import ActionError
+from .ticket import AbstractTicket
+from .rest import AbstractREST
+
+
+class Zammad(AbstractTicket, AbstractREST):
+    """Zammad.
+
+    Managing Tickets using Zammad.
+    """
+
+    auth_type = "apikey"
+    token_type = "Bearer"
+    article_base = {"type": "note", "internal": False}
+    data_format: str = "raw"
+
+    def __init__(self, *args, **kwargs):
+        super(Zammad, self).__init__(*args, **kwargs)
+        self.auth = {"apikey": ZAMMAD_TOKEN}
+
+    async def close(self):
+        pass
+
+    async def open(self):
+        pass
+
+    async def create(self, hook, *args, **kwargs):
+        """create.
+
+        Create a new Ticket.
+        """
+        self.url = f"{ZAMMAD_INSTANCE}api/v1/tickets"
+        self.method = "post"
+        group = kwargs.pop("group", ZAMMAD_DEFAULT_GROUP)
+        title = kwargs.pop("title", None)
+        customer = kwargs.pop("customer", ZAMMAD_DEFAULT_CUSTOMER)
+        article = kwargs.pop("article")
+        article = {**self.article_base, **article}
+        data = {
+            "title": title,
+            "group": group,
+            "customer": customer,
+            "article": article,
+        }
+        try:
+            result, _ = await self.request(self.url, self.method, data=data)
+            return result
+        except Exception as e:
+            raise ActionError(f"Error creating Zammad Ticket: {e}") from e

flowtask/hooks/hook.py

@@ -0,0 +1,62 @@
+from uuid import uuid4
+from navconfig.logging import logging
+from ..exceptions import ConfigError
+from .step import StepAction, import_component
+
+
+class Hook:
+    """Hook.
+
+    Compile a Hook (Triggers and Actions) and got every step on the hook.
+    """
+
+    def __init__(self, hook: dict):
+        self.triggers: list = []
+        self.logger = logging.getLogger(name='Flowtask.Hook')
+        self._id = hook.pop("id", uuid4())
+        self.name = hook.pop("name")
+        try:
+            triggers = hook["When"]
+        except KeyError as exc:
+            raise ConfigError(
+                "Hook Error: Unable to find Trigger: *When* parameter"
+            ) from exc
+        try:
+            actions = hook["Then"]
+        except KeyError as exc:
+            raise ConfigError(
+                "Hook Error: Unable to get list of Actions: *Then* parameter"
+            ) from exc
+        ## build Hook Component:
+        self.build(triggers, actions)
+
+    def build(self, triggers: list, actions: list):
+        self._actions: list = []
+        # "Then": Load Actions
+        for step in actions:
+            for step_name, params in step.items():
+                action = StepAction(step_name, params)
+                self._actions.append(action)
+        for step in triggers:
+            # When: Load Triggers
+            # Triggers are listener for events.
+            for step_name, params in step.items():
+                trigger_cls = import_component(step_name, "flowtask.hooks.types", "types")
+                if trigger_cls:
+                    # start trigger:
+                    args = {"name": self.name, "actions": self._actions}
+                    args = {**args, **params}
+                    try:
+                        hook = trigger_cls(**args)
+                        self.triggers.append(hook)
+                        self.logger.debug(
+                            f":: Loading Hook {self.name}"
+                        )
+                    except Exception as exc:
+                        self.logger.error(
+                            f"Unable to load Trigger {step_name}: {exc}"
+                        )
+                else:
+                    self.logger.warning(
+                        f"Unable to load Trigger: {step_name}"
+                    )

flowtask/hooks/models.py

@@ -0,0 +1,17 @@
+from datetime import datetime
+from dataclasses import InitVar
+from datamodel import Field
+from asyncdb.models import Model
+from uuid import UUID, uuid4
+
+class HookObject(Model):
+    trigger_id: UUID = Field(required=True, primary_key=True, default=uuid4, db_default="auto")
+    name: str = Field(required=True)
+    definition: dict = Field(required=True)
+    created_by: int = Field(required=False)
+    created_at: datetime = Field(required=False, default=datetime.now())
+
+    class Meta:
+        name: str = "triggers"
+        schema: str = "navigator"
+        strict: bool = False

flowtask/hooks/service.py

@@ -0,0 +1,187 @@
+"""
+Hook Management.
+
+This module is responsible for handling the Hooks in Flowtask.
+"""
+import asyncio
+from navconfig.logging import logging
+from asyncdb import AsyncDB
+from navigator.types import WebApp
+from navigator.applications.base import BaseApplication
+from navigator.views import ModelView
+from ..exceptions import ConfigError, FlowTaskError
+from ..conf import TASK_STORAGES, default_dsn
+from .types.base import BaseTrigger
+from .hook import Hook
+from .models import HookObject
+
+
+class HookHandler(ModelView):
+    model = HookObject
+    name = "Navigator Triggers"
+    pk: str = "trigger_id"
+
+    async def _set_created_by(self, value, column, data):
+        return await self.get_userid(session=self._session)
+
+    @ModelView.service_auth
+    async def _post_data(self, *args, **kwargs):
+        payload = await super()._post_data(*args, **kwargs)
+        if not payload:
+            raise ConfigError("Trigger: No payload provided.")
+        payload['definition'] = payload.copy()
+        # then, remove the components
+        del payload['When']
+        del payload['Then']
+        # and renamed "id" to "trigger_id":
+        payload['trigger_id'] = payload.pop('id')
+        return payload
+
+    @ModelView.service_auth
+    async def _post_response(
+        self,
+        response,
+        fields: list = None,
+        headers: dict = None,
+        status: int = 200
+    ):
+        if status == 201:
+            # Hook was correctly created.
+            # TODO: remove the previous when edited.
+            try:
+                hookservice = self.request.app['HookService']
+                hook = Hook(hook=response.definition)
+                for trigger in hook.triggers:
+                    tg = trigger()
+                    print('TG > ', tg)
+                    hookservice.add_hook(tg)
+                    await tg.start()
+                message = f"Hook {response.trigger_id} added successfully."
+                self.logger.info(
+                    message
+                )
+                response.status = message
+            except Exception as exc:
+                self.logger.error(
+                    f"Error adding hook: {exc}"
+                )
+        return self.json_response(
+            response=response,
+            status=status
+        )
+
+class HookService:
+    def __init__(self, event_loop: asyncio.AbstractEventLoop, app: BaseApplication, **kwargs):
+        self._loop = event_loop
+        self._hooks: list = []
+        self._started: bool = False
+        self.logger = logging.getLogger(name="Flowtask.HookService")
+        self.app: WebApp = None
+        # TaskStorage
+        self._storage = kwargs.pop("storage", "default")
+        # App
+        if isinstance(app, BaseApplication):  # migrate to BaseApplication (on types)
+            self.app = app.get_app()
+        elif isinstance(app, WebApp):
+            self.app = app  # register the app into the Extension
+        else:
+            raise TypeError(
+                f"Invalid type for Application Setup: {app}:{type(app)}"
+            )
+        try:
+            self.taskstore = TASK_STORAGES[self._storage]
+        except KeyError as exc:
+            raise RuntimeError(
+                f"Invalid Task Storage > {self._storage}"
+            ) from exc
+        if not self.taskstore.path:
+            raise ConfigError(
+                f"Current Task Storage {self.taskstore!r} is not Supported for saving Triggers/Hooks."
+            )
+
+    def add_hook(self, hook: BaseTrigger):
+        """
+        Add the Hook, start it and calling the Setup.
+        """
+        self._hooks.append(hook)
+        hook.setup(app=self.app)
+
+    add_trigger = add_hook
+
+    async def setup(self) -> None:
+        """setup.
+
+            Service Configuration.
+        Args:
+            app (aiohttp.web.Application): Web Application.
+        """
+        self.app['HookService'] = self
+        # Start all hooks, and register the endpoint.
+        await self.start_hooks(self.app)
+        # register endpoint to add Triggers to HookService.
+        HookHandler.configure(
+            self.app,
+            path='/api/v1/triggers/'
+        )
+        self.logger.notice('Hook Service Started.')
+
+    async def start_hooks(self, app: WebApp) -> None:
+        """start_hooks.
+
+            Starts the Hook Service.
+        """
+        # First: loading the File-based Hooks
+        await self.load_fs_hooks()
+        # Second: loading the Database Hooks
+        await self.load_db_hooks()
+        # mark service as started.
+        self._started = True
+
+    async def load_fs_hooks(self) -> None:
+        """load_fs_hooks.
+
+        Load all Hooks from the Task Storage (Filesystem).
+        """
+        self.logger.notice(":: Loading Hooks from Filesystem.")
+        for program_dir in self.taskstore.path.iterdir():
+            if program_dir.is_dir():
+                hooks_dir = program_dir.joinpath("hooks.d")
+                for file_path in hooks_dir.rglob("*.*"):
+                    # only work with supported extensions:
+                    if file_path.suffix in ('.json', '.yaml', '.yml'):
+                        try:
+                            store = await self.taskstore.open_hook(file_path)
+                        except Exception as exc:
+                            self.logger.warning(
+                                f"Unable to load Hook {file_path!r}: Invalid Hook File, {exc}"
+                            )
+                            continue
+                        try:
+                            hook = Hook(hook=store)
+                            if hook:
+                                for trigger in hook.triggers:
+                                    self.add_hook(trigger)
+                        except FlowTaskError:
+                            pass
+
+    async def load_db_hooks(self) -> None:
+        """load_db_hooks.
+
+        Load all Hooks saved into the Database.
+        """
+        db = AsyncDB('pg', dsn=default_dsn)
+        async with await db.connection() as conn:
+            HookObject.Meta.connection = conn
+            hooks = await HookObject.all()
+            for hook in hooks:
+                definition = hook.definition
+                try:
+                    hook = Hook(hook=definition)
+                    for trigger in hook.triggers:
+                        self.add_hook(trigger)
+                except FlowTaskError:
+                    pass
+                except Exception as exc:
+                    self.logger.error(
+                        f"Error Loading Database Hooks: {exc}"
+                    )

flowtask/hooks/step.py

@@ -0,0 +1,91 @@
+from typing import Union, Any
+from collections.abc import Callable, Awaitable
+import importlib
+from navconfig.logging import logging
+from ..exceptions import ConfigError, FlowTaskError, ComponentError, NotSupported
+from ..components import getComponent, importComponent
+from ..components.abstract import AbstractFlow
+from .actions.abstract import AbstractAction
+
+
+_ACTIONS: dict[str, Any] = {}
+
+
+def import_component(component: str, classpath: str, package: str = "components"):
+    module = importlib.import_module(classpath, package=package)
+    # module = importlib.import_module(f".{name.lower()}", package=module_path)
+    obj = getattr(module, component, None)
+    return obj
+
+
+class StepAction:
+    def __init__(self, action: str, params: dict, **kwargs) -> None:
+        self.name = action
+        self._step: Union[Callable, Awaitable] = None
+        try:
+            # Check if the action is already loaded
+            if hasattr(self, "_action"):
+                return
+            # Check if the action is a function or a class
+            action_cls = None
+            if callable(action):
+                self._action = action
+            elif _ACTIONS.get(action):
+                action_cls = _ACTIONS[action]
+            else:
+                # Load the action from the components module
+                try:
+                    action_cls = importComponent(action)
+                except (NotSupported, ImportError):
+                    pass
+            if not action_cls:
+                # Load the action from the actions module
+                action_cls: AbstractAction = import_component(
+                    action,
+                    "flowtask.hooks.actions",
+                    "actions"
+                )
+                if not action_cls:
+                    raise ConfigError(
+                        f"Unable to load Action: {action}"
+                    )
+            args = {**kwargs, **params}
+            self._action = action_cls(**args)
+            # saving into _ACTIONS variable
+            _ACTIONS[action] = action_cls
+        except (ImportError, RuntimeError) as exc:
+            raise FlowTaskError(
+                f"Unable to load Action {action}: {exc}"
+            ) from exc
+        self.params = args
+
+    def __repr__(self) -> str:
+        return f"<StepAction.{self.name}: {self.params!r}>"
+
+    @property
+    def component(self):
+        return self._action
+
+    async def run(self, hook, *args, **kwargs):
+        """Run action involved"""
+        try:
+            try:
+                async with self._action as step:
+                    # Check if Step is a AbstractFlow to set the "self.data" property
+                    if isinstance(step, AbstractFlow):
+                        # Passing all arguments to the Flowtask component
+                        step.data = kwargs
+                        step.result = kwargs.get('result')
+                        result = await step.run()
+                    else:
+                        result = await step.run(hook, *args, **kwargs)
+                return result
+            except Exception as exc:
+                logging.error(
+                    f"Error running action {self._action!s}: {exc}"
+                )
+        except Exception as exc:
+            logging.error(
+                f"Unable to load Action {self._action}: {exc}"
+            )
+            raise

flowtask/hooks/types/__init__.py

@@ -0,0 +1,23 @@
+"""
+Hook Types.
+
+Hook Types are triggers that can be executed when an event is triggered.
+
+1. FSWatchdog: Watches a specified directory for changes.
+"""
+from .fs import FSWatchdog
+from .web import WebHook
+from .imap import IMAPWatchdog
+from .tagged import TaggedIMAPWatchdog
+from .ssh import SFTPWatchdog
+from .upload import UploadHook
+from .postgres import PostgresTrigger
+from .http import HTTPHook
+from .sharepoint import SharePointTrigger
+from .mail import EmailTrigger
+from .jira import JiraTrigger
+# Message Brokers:
+from .brokers.mqtt import MQTTTrigger
+from .brokers.sqs import SQSTrigger
+from .brokers.redis import RedisTrigger
+from .brokers.rabbitmq import RabbitMQTrigger

flowtask/hooks/types/base.py

@@ -0,0 +1,129 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Optional
+from collections.abc import Callable
+import asyncio
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from navconfig import config
+from navconfig.logging import logging
+from navigator.types import WebApp
+from .responses import TriggerResponse
+from ..actions import AbstractAction
+from ...interfaces import (
+    MaskSupport,
+    LocaleSupport,
+    LogSupport
+)
+
+
+class BaseTrigger(MaskSupport, LogSupport, LocaleSupport, ABC):
+    """BaseTrigger.
+
+        Base class for all Triggers in FlowTask.
+    """
+    # Signal for startup/shutdown method for this Hook (using aiohttp signals)
+    on_startup: Optional[Callable] = None
+    on_shutdown: Optional[Callable] = None
+
+    def __init__(self, *args, actions: list = None, **kwargs):
+        self.trigger_id = kwargs.get("id", uuid.uuid4())
+        super().__init__(*args, **kwargs)
+        self._name_ = self.__class__.__name__
+        self.description = kwargs.pop('description', self._name_)
+        self._args = args
+        self._kwargs = kwargs
+        self.app: WebApp = None
+        self._logger = logging.getLogger(
+            f"Trigger.{self._name_}"
+        )
+        self._response: TriggerResponse = None
+        self._environment = config
+        self._actions: list[AbstractAction] = actions
+        for key, val in kwargs.items():
+            setattr(self, key, val)
+
+    def setup(self, app: WebApp) -> None:
+        """setup.
+
+            Configuration of Trigger when started.
+        Args:
+            app (aiohttp.web.Application): Web Application.
+        """
+        self.app = app  # register the app into the Extension
+        # startup operations over extension backend
+        try:
+            # avoid Cannot modify frozen list
+            if callable(self.on_startup):
+                app.on_startup.append(self.on_startup)
+
+            if callable(self.on_shutdown):
+                app.on_shutdown.append(self.on_shutdown)
+        except Exception as e:
+            self._logger.warning(
+                f"Error setting up Trigger {self.__class__.__name__}: {e}"
+            )
+            return False
+
+    def get_env_value(self, key, default: str = None):
+        if val := os.getenv(key):
+            return val
+        elif val := self._environment.get(key, default):
+            return val
+        else:
+            return key
+
+    def add_action(self, action: AbstractAction):
+        self._actions.append(action)
+
+    def get_actions(self) -> list:
+        return self._actions
+
+    async def run_actions(self, *args, **kwargs):
+        result = None
+        if self._actions:
+            for action in self._actions:
+                self._logger.notice(
+                    f"Calling Action: {action}"
+                )
+                try:
+                    result = await action.run(
+                        hook=self,
+                        result=result,
+                        *args,
+                        **kwargs
+                    )
+                except Exception as e:
+                    # Handle any exceptions that might occur during action.run()
+                    self._logger.error(
+                        f"Error while running {action}: {e}"
+                    )
+            return result
+        else:
+            self._logger.warning(
+                f"Trigger {self.__class__.__name__}: No actions were found to be executed."
+            )
+
+    def call_actions(self, *args, **kwargs):
+        # Run the actions in a thread pool
+        _new = False
+        try:
+            loop = asyncio.get_event_loop()
+            _new = False
+        except RuntimeError:
+            loop = asyncio.new_event_loop()
+            _new = True
+        try:
+            with ThreadPoolExecutor() as executor:
+                future = executor.submit(
+                    loop.run_until_complete, self.run_actions(*args, **kwargs)
+                )
+                future.result()
+        except Exception as exc:
+            self._logger.error(f"Error Calling Action: {exc}")
+        finally:
+            if _new is True:
+                try:
+                    loop.close()
+                except RuntimeError:
+                    pass

flowtask/hooks/types/brokers/__init__.py

@@ -0,0 +1,11 @@
+"""
+Message Brokers.
+
+Message Brokers are services that allow applications to send and receive messages.
+
+Supported Message Brokers on Flowtask are:
+- MQTT
+- SQS
+- Redis
+- RabbitMQ
+"""