flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/hooks/types/brokers/base.py

@@ -0,0 +1,54 @@
+import asyncio
+from typing import Optional, Any
+from aiohttp import web
+from abc import abstractmethod
+from ..base import BaseTrigger
+
+class BaseMQTrigger(BaseTrigger):
+    def __init__(self, *args, actions: list = None, **kwargs):
+        super().__init__(*args, actions=actions, **kwargs)
+        self.consumer_task: Optional[asyncio.Task] = None
+        self._connection = None  # To be set in subclass
+        self._queue_name = kwargs.get('queue_name', 'default_queue')
+
+    @abstractmethod
+    async def connect(self):
+        """Establish the connection."""
+        pass
+
+    @abstractmethod
+    async def disconnect(self):
+        """Disconnect the connection."""
+        pass
+
+    async def on_startup(self, app):
+        self._logger.info(
+            f"Starting MQ Broker {self.__class__.__name__}"
+        )
+        await self.connect()
+        self.consumer_task = asyncio.create_task(self.start_consuming())
+
+    async def on_shutdown(self, app):
+        self._logger.info(
+            f"Shutting down MQ Broker {self.__class__.__name__}"
+        )
+        if self.consumer_task:
+            self.consumer_task.cancel()
+            try:
+                await self.consumer_task
+            except asyncio.CancelledError:
+                self._logger.info("Consumer task cancelled.")
+            self.consumer_task = None
+        await self.disconnect()
+
+    @abstractmethod
+    async def start_consuming(self):
+        """Start consuming messages."""
+        pass
+
+    def setup(self, app: web.Application):
+        super().setup(app)
+
+    @abstractmethod
+    async def _consumer_callback(self, *args, **kwargs) -> None:
+        pass

flowtask/hooks/types/brokers/mqtt.py

@@ -0,0 +1,35 @@
+import asyncio
+from gmqtt import Client as MQTTClient
+from ..base import BaseTrigger
+
+class MQTTTrigger(BaseTrigger):
+    def __init__(self, *args, broker_url: str, topics: list, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.broker_url = broker_url
+        self.topics = topics
+        self.client = MQTTClient(client_id='')
+        self.client.on_message = self.on_message
+        self.client.on_connect = self.on_connect
+        self.client.on_disconnect = self.on_disconnect
+
+    async def on_startup(self, app):
+        self._logger.info(f"Connecting to MQTT broker at {self.broker_url}")
+        await self.client.connect(self.broker_url)
+
+    async def on_shutdown(self, app):
+        self._logger.info("Disconnecting from MQTT broker")
+        await self.client.disconnect()
+
+    def on_connect(self, client, flags, rc, properties):
+        self._logger.info("Connected to MQTT broker")
+        for topic in self.topics:
+            client.subscribe(topic)
+            self._logger.info(f"Subscribed to topic: {topic}")
+
+    def on_disconnect(self, client, packet, exc=None):
+        self._logger.warning("Disconnected from MQTT broker")
+        # Implement reconnection logic if needed
+
+    def on_message(self, client, topic, payload, qos, properties):
+        self._logger.info(f"Received message on topic {topic}: {payload}")
+        asyncio.create_task(self.run_actions(topic=topic, payload=payload))

flowtask/hooks/types/brokers/rabbitmq.py

@@ -0,0 +1,82 @@
+from typing import Optional, Any
+import aiormq
+from navigator.brokers.rabbitmq import RabbitMQConnection
+from .base import BaseMQTrigger
+
+
+class RabbitMQTrigger(BaseMQTrigger):
+    def __init__(
+        self,
+        *args,
+        queue_name: str,
+        routing_key: str = '',
+        exchange_name: str = '',
+        exchange_type: str = 'topic',
+        credentials: Optional[str] = None,
+        actions: list = None,
+        **kwargs
+    ):
+        super().__init__(
+            *args,
+            actions=actions,
+            queue_name=queue_name,
+            **kwargs
+        )
+        self._queue_name = queue_name
+        self._routing_key = routing_key
+        self._exchange_name = exchange_name
+        self._exchange_type = exchange_type
+        self._connection = RabbitMQConnection(
+            credentials=credentials
+        )
+        self.prefetch_count = kwargs.get('prefetch_count', 1)
+
+    async def connect(self):
+        await self._connection.connect()
+        self._logger.info("RabbitMQ connection established.")
+        # Ensure the exchange exists
+        await self._connection.ensure_exchange(
+            exchange_name=self._exchange_name or self._queue_name,
+            exchange_type=self._exchange_type
+        )
+        # Ensure the queue exists and bind it to the exchange
+        await self._connection.ensure_queue(
+            queue_name=self._queue_name,
+            exchange_name=self._exchange_name or self._queue_name,
+            routing_key=self._routing_key
+        )
+
+    async def disconnect(self):
+        await self._connection.disconnect()
+        self._logger.info("RabbitMQ connection closed.")
+
+    async def start_consuming(self):
+        await self._connection.consume_messages(
+            queue_name=self._queue_name,
+            callback=self._consumer_callback,
+            prefetch_count=self.prefetch_count
+        )
+
+    async def _consumer_callback(
+        self,
+        message: aiormq.abc.DeliveredMessage,
+        body: Any
+    ) -> None:
+        """
+        Callback from Consumer.
+
+        Used to call actions based on consumer messages received.
+        """
+        try:
+            message_id = message.delivery_tag
+            self._logger.info(f"Received Message ID: {message_id} Body: {body}")
+            # Call run_actions with the received message
+            await self.run_actions(
+                message_id=message_id,
+                payload=message,
+                message=body
+            )
+        except Exception as e:
+            self._logger.error(f"Error in _consumer_callback: {e}")
+            # Handle message rejection or requeueing if necessary
+            await self._connection.reject_message(message, requeue=False)

flowtask/hooks/types/brokers/redis.py

@@ -0,0 +1,83 @@
+from typing import Optional, Any
+from navigator.brokers.redis import RedisConnection
+from .base import BaseMQTrigger
+
+
+class RedisTrigger(BaseMQTrigger):
+    """Redis Trigger.
+
+    Trigger that listens to a Redis Stream and calls actions based on the received messages.
+    """
+    def __init__(
+        self,
+        *args,
+        stream_name: str,
+        group_name: str = 'default_group',
+        consumer_name: str = 'default_consumer',
+        credentials: Optional[dict] = None,
+        actions: list = None,
+        **kwargs
+    ):
+        super().__init__(
+            *args,
+            actions=actions,
+            queue_name=stream_name,
+            **kwargs
+        )
+        self._stream_name = stream_name
+        self._group_name = group_name
+        self._consumer_name = consumer_name
+        self._connection = RedisConnection(
+            credentials=credentials,
+            group_name=group_name,
+            consumer_name=consumer_name,
+            queue_name=stream_name
+        )
+        self.count = kwargs.get('count', 1)
+        self.block = kwargs.get('block', 1000)
+
+    async def connect(self):
+        await self._connection.connect()
+        self._logger.info(
+            "Redis connection established."
+        )
+
+    async def disconnect(self):
+        await self._connection.disconnect()
+        self._logger.info(
+            "Redis connection closed."
+        )
+
+    async def start_consuming(self):
+        await self._connection.consume_messages(
+            queue_name=self._stream_name,
+            callback=self._consumer_callback,
+            count=self.count,
+            block=self.block,
+            consumer_name=self._consumer_name
+        )
+
+    async def _consumer_callback(
+        self,
+        data: dict,
+        processed_message: Any
+    ) -> None:
+        """
+        Callback from Consumer.
+
+        Used to call actions based on consumer messages received.
+        """
+        try:
+            message_id = data.get('message_id')
+            self._logger.info(
+                f"Received Message ID: {message_id} Body: {processed_message}"
+            )
+            # Call run_actions with the received message
+            await self.run_actions(
+                payload=data,
+                message_id=message_id,
+                message=processed_message
+            )
+        except Exception as e:
+            self._logger.error(f"Error in _consumer_callback: {e}")
+            raise

flowtask/hooks/types/brokers/sqs.py

@@ -0,0 +1,44 @@
+from typing import Optional, Any
+from navigator.brokers.sqs import SQSConnection
+from .base import BaseMQTrigger
+
+class SQSTrigger(BaseMQTrigger):
+    def __init__(
+        self,
+        *args,
+        queue_name: str,
+        credentials: Optional[dict] = None,
+        actions: list = None,
+        **kwargs
+    ):
+        super().__init__(
+            *args,
+            actions=actions,
+            queue_name=queue_name,
+            **kwargs
+        )
+        self._connection = SQSConnection(credentials=credentials)
+        self.max_messages = kwargs.get('max_messages', 10)
+        self.wait_time = kwargs.get('wait_time', 10)
+        self.idle_sleep = kwargs.get('idle_sleep', 5)
+
+    async def connect(self):
+        await self._connection.connect()
+        self._logger.info(
+            "AWS SQS connection established."
+        )
+
+    async def disconnect(self):
+        await self._connection.disconnect()
+        self._logger.info(
+            "AWS SQS connection closed."
+        )
+
+    async def start_consuming(self):
+        await self._connection.consume_messages(
+            queue_name=self._queue_name,
+            callback=self._consumer_callback,
+            max_messages=self.max_messages,
+            wait_time=self.wait_time,
+            idle_sleep=self.idle_sleep
+        )

flowtask/hooks/types/fs.py

@@ -0,0 +1,232 @@
+import time
+import os
+from collections import defaultdict
+from navconfig.logging import logging
+from watchdog.observers import Observer
+from watchdog.events import PatternMatchingEventHandler
+from navigator.types import WebApp
+from ...exceptions import ComponentError
+from .watch import BaseWatcher, BaseWatchdog
+
+
+# TODO> PatternMatchingEventHandler
+
+fslog = logging.getLogger("watchdog.observers").setLevel(logging.WARNING)
+
+
+class FsHandler(PatternMatchingEventHandler):
+    def __init__(self, parent: BaseWatchdog, patterns=None, *args, **kwargs):
+        self.not_empty = kwargs.pop("not_empty", False)
+        super().__init__(patterns=patterns, *args, **kwargs)
+        self.debounced_events = defaultdict(lambda: 0)
+        self.parent = parent
+        self.recently_created = set()
+        self._logger = logging.getLogger("Watcher.FS")
+
+    def zero_size(self, filepath: str):
+        """Check if the file is of zero size."""
+        try:
+            return os.path.getsize(filepath) == 0
+        except (OSError, FileNotFoundError):
+            # Handle cases where the file no longer exists
+            return True
+
+    def process(self, event):
+        """Process the event if it matches the patterns and isn't a directory."""
+        if event.is_directory:
+            return None
+
+        # Check if an event for this path has been triggered recently
+        last_event_time = self.debounced_events[event.src_path]
+        current_time = time.time()
+        if current_time - last_event_time < 0.5:  # 0.5 seconds debounce time
+            return
+
+        self.debounced_events[event.src_path] = current_time
+
+    def on_created(self, event):
+        if event.is_directory:
+            return
+        if "created" not in self.parent.events:
+            return
+        if self.not_empty and self.zero_size(event.src_path):
+            self._logger.warning(
+                f"File {event.src_path} has zero size and 'not_empty' is True. Skipping."
+            )
+            return  # Skip triggering actions
+        print(f"Watchdog received created event - {event.src_path} s.")
+        self.recently_created.add(event.src_path)  # recently created
+        self.process(event)
+        # after, running actions:
+        args = {
+            "directory": self.parent.directory,
+            "event": event,
+            "on": "created",
+            "filename": event.src_path,
+        }
+        self.parent.call_actions(**args)
+
+    def on_modified(self, event):
+        """
+        Handle file modification events.
+
+        This method is called when a file modification event is detected. It processes
+        the event and calls the appropriate actions if the event meets certain criteria.
+
+        Parameters:
+        event (FileSystemEvent): The event object containing information about the
+                                 file modification.
+
+        Returns:
+        None
+
+        Note:
+        - The method returns early if the event is for a directory or if 'modified'
+          is not in the parent's events list.
+        - It also returns early if the file was recently created to avoid duplicate processing.
+        - If the file has zero size, a warning is logged.
+        - The event is processed and actions are called twice with the same arguments.
+        """
+        if event.is_directory:
+            return
+        if "modified" not in self.parent.events:
+            return
+        if event.src_path in self.recently_created:
+            self.recently_created.remove(event.src_path)
+            return
+        if self.not_empty and self.zero_size(event.src_path):
+            self._logger.warning(
+                f"File {event.src_path} has zero size and 'not_empty' is True. Skipping."
+            )
+            return  # Skip triggering actions
+        print(f"Watchdog received modified event - {event.src_path} s.")
+        self.process(event)
+        args = {
+            "directory": self.parent.directory,
+            "event": event,
+            "on": "modified",
+            "filename": event.src_path,
+        }
+        self.parent.call_actions(**args)
+        args = {
+            "directory": self.parent.directory,
+            "event": event,
+            "on": "created",
+            "filename": event.src_path,
+        }
+        self.parent.call_actions(**args)
+
+    def on_moved(self, event):
+        if event.is_directory:
+            return
+        if "moved" not in self.parent.events:
+            return
+        # For moved events, check the destination path
+        if self.not_empty and self.zero_size(event.dest_path):
+            self._logger.warning(f"File {event.dest_path} has zero size and 'not_empty' is True. Skipping.")
+            return  # Skip triggering actions
+        print(f"Watchdog received moved event - from {event.src_path} to {event.dest_path}.")
+        self.process(event)
+        args = {
+            "directory": self.parent.directory,
+            "event": event,
+            "on": "moved",
+            "filename": event.dest_path,
+        }
+        self.parent.call_actions(**args)
+
+    def on_deleted(self, event):
+        if "deleted" not in self.parent.events:
+            return
+        print(f"Watchdog received deleted event - {event.src_path} s.")
+        self.process(event)
+        args = {
+            "directory": self.parent.directory,
+            "event": event,
+            "on": "deleted",
+            "filename": event.src_path,
+        }
+        self.parent.call_actions(**args)
+
+
+class FsWatcher(BaseWatcher):
+    def __init__(self, pattern, *args, **kwargs):
+        super(FsWatcher, self).__init__(*args, **kwargs)
+        self.directory = kwargs.pop("directory", None)
+        self.filename = kwargs.pop("filename", [])
+        self.recursive = kwargs.pop("recursive", True)
+        self.not_empty = kwargs.pop("not_empty", False)
+        self.observer = Observer()
+        self._patterns = pattern
+
+    def run(self):
+        event_handler = FsHandler(
+            parent=self.parent,
+            patterns=self._patterns,
+            not_empty=self.not_empty
+        )
+        self.observer.schedule(
+            event_handler,
+            self.directory,
+            recursive=self.recursive
+        )
+        self.observer.start()
+        try:
+            while not self.stop_event.is_set():
+                time.sleep(self.timeout)
+        except KeyboardInterrupt:
+            self.stop()
+            print("Watchdog FS Observer was stopped")
+        except Exception as e:
+            self.stop()
+            raise e
+
+    def stop(self):
+        try:
+            self.observer.stop()
+        except Exception:
+            pass
+        self.observer.join()
+
+
+class FSWatchdog(BaseWatchdog):
+    """FSWatchdog.
+    Checking for changes in the filesystem and dispatch events.
+    """
+
+    timeout: int = 5
+    recursive: bool = True
+
+    def create_watcher(self, *args, **kwargs) -> BaseWatcher:
+        self.recursive = kwargs.pop("recursive", False)
+        self.events = kwargs.pop("on", ["created", "modified", "deleted", "moved"])
+        self.filename = kwargs.pop("filename", [])
+        self.not_empty = kwargs.pop("not_empty", False)
+        if not self.filename:
+            self.filename = ["*"]
+        else:
+            self.filename = [f"*{filename}" for filename in self.filename]
+        try:
+            self.directory = kwargs["directory"]
+        except KeyError as exc:
+            raise ComponentError("Unable to load Directory on FSWatchdog") from exc
+        return FsWatcher(
+            pattern=self.filename,
+            directory=self.directory,
+            timeout=self.timeout,
+            recursive=self.recursive,
+            not_empty=self.not_empty,
+        )
+
+    async def on_startup(self, app: WebApp = None) -> None:
+        # Perform permissions check
+        if not os.access(self.directory, os.R_OK | os.X_OK):
+            raise PermissionError(
+                f"Insufficient permissions to access directory '{self.directory}'. "
+                "Read and execute permissions are required."
+            )
+        # Start the watcher
+        self.watcher.start()
+
+    async def on_shutdown(self, app: WebApp = None) -> None:
+        self.watcher.stop()

flowtask/hooks/types/http.py

@@ -0,0 +1,49 @@
+from navigator.views import BaseHandler
+from navigator.libs.json import JSONContent
+from navigator.types import WebApp
+from .base import BaseTrigger
+
+
+class HTTPHook(BaseTrigger, BaseHandler):
+    """HTTPHook.
+
+    Base Hook for all HTTP-based hooks.
+
+    """
+    methods: list = ["GET", "POST"]
+    default_status: int = 202
+
+    def __init__(self, *args, **kwargs):
+        self.method: str = kwargs.pop('method', None)
+        if self.method:
+            self.methods = [self.method]
+        super(HTTPHook, self).__init__(*args, **kwargs)
+        trigger_url = kwargs.get('url', None)
+        if trigger_url:
+            self.url = trigger_url
+        else:
+            self._base_url = kwargs.get('base_url', '/api/v1/webhook/')
+            self.url = f"{self._base_url}{self.trigger_id}"
+        self._json = JSONContent()
+
+    def setup(self, app: WebApp) -> None:
+        super().setup(app)
+        self._logger.notice(
+            f"Set the unique URL Trigger to: {self.url}"
+        )
+        if hasattr(self, 'handle'):
+            self.app.router.add_route(
+                'POST',
+                self.url,
+                self.handle
+            )
+        else:
+            # Class-based Views
+            for method in self.methods:
+                handler = getattr(self, method.lower(), None)
+                if handler:
+                    self.app.router.add_route(
+                        method,
+                        self.url,
+                        handler
+                    )