flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/hooks/types/postgres.py

@@ -0,0 +1,98 @@
+from typing import Optional
+from dataclasses import dataclass
+import asyncio
+import asyncpg
+import backoff  # For exponential backoff
+from navconfig.logging import logging
+from navigator.conf import default_dsn  # Your default DSN configuration
+from .base import BaseTrigger  # Base class for triggers
+
+
+@dataclass
+class Notification:
+    channel: str
+    payload: str
+    pid: str
+
+
+class PostgresTrigger(BaseTrigger):
+    """PostgresTrigger.
+
+    Trigger that listens for PostgreSQL events and sends notifications.
+    Using LISTEN/NOTIFY Infraestructure.
+    """
+    def __init__(
+        self,
+        *args,
+        dsn: Optional[str] = None,
+        channel: str = 'notifications',
+        **kwargs
+    ):
+        super().__init__(*args, **kwargs)
+        self.dsn = dsn or default_dsn
+        self.channel = channel
+        self.connection: Optional[asyncpg.Connection] = None
+        self._listening_task: Optional[asyncio.Task] = None
+        self._reconnecting = False
+
+    async def on_startup(self, app):
+        self._logger.info(
+            f"Starting PostgresTrigger listening on channel '{self.channel}'"
+        )
+        await self.connect()
+
+    async def on_shutdown(self, app):
+        self._logger.info("Shutting down PostgresTrigger")
+        if self._listening_task:
+            self._listening_task.cancel()
+        if self.connection:
+            await self.connection.close()
+
+    @backoff.on_exception(
+        backoff.expo,
+        (asyncpg.exceptions.PostgresError, ConnectionError),
+        max_tries=5,
+        on_backoff=lambda details: logging.warning(f"Retrying connection in {details['wait']} seconds...")
+    )
+    async def connect(self):
+        self.connection = await asyncpg.connect(dsn=self.dsn)
+        await self.connection.add_listener(
+            self.channel,
+            self.notification_handler
+        )
+        self._logger.info(
+            f"Connected to PostgreSQL and listening on channel '{self.channel}'"
+        )
+        self._listening_task = asyncio.create_task(self.keep_listening())
+
+    async def keep_listening(self):
+        try:
+            while True:
+                await asyncio.sleep(3600)  # Keep the coroutine alive
+        except asyncio.CancelledError:
+            self._logger.info(
+                "Listening task cancelled"
+            )
+        except Exception as e:
+            self._logger.error(
+                f"Error in keep_listening: {e}"
+            )
+            await self.reconnect()
+
+    async def reconnect(self):
+        if not self._reconnecting:
+            self._reconnecting = True
+            if self.connection:
+                await self.connection.close()
+            self._logger.info(
+                "Attempting to reconnect to PostgreSQL..."
+            )
+            await self.connect()
+            self._reconnecting = False
+
+    async def notification_handler(self, connection, pid, channel, payload):
+        self._logger.info(
+            f"Received notification on channel '{channel}': {payload}"
+        )
+        notification = Notification(channel=channel, payload=payload, pid=pid)
+        await self.run_actions(notification=notification)

flowtask/hooks/types/responses/__init__.py

@@ -0,0 +1,8 @@
+"""
+Responses.
+
+Pre-defined responses for Hooks.
+"""
+from .base import TriggerResponse
+
+__all__ = ("TriggerResponse",)

flowtask/hooks/types/responses/base.py

@@ -0,0 +1,5 @@
+class TriggerResponse:
+    """Basic Response.
+
+    Basic Reponse dispatched by a Hook.
+    """

flowtask/hooks/types/sharepoint.py

@@ -0,0 +1,288 @@
+from typing import Any, Optional
+from collections.abc import Callable
+import asyncio
+import datetime
+from urllib.parse import quote
+import aiohttp
+from aiohttp import web
+from azure.identity.aio import ClientSecretCredential
+from msgraph import GraphServiceClient, GraphRequestAdapter
+from msgraph.generated.models.subscription import Subscription
+from kiota_authentication_azure.azure_identity_authentication_provider import AzureIdentityAuthenticationProvider
+from navconfig.logging import logging
+from .http import HTTPHook
+from ...conf import (
+    SHAREPOINT_APP_ID,
+    SHAREPOINT_APP_SECRET,
+    SHAREPOINT_TENANT_ID,
+    SHAREPOINT_DEFAULT_HOST,
+    SHAREPOINT_TENANT_NAME
+)
+
+logging.getLogger(name='azure.identity.aio').setLevel(logging.WARNING)
+logging.getLogger(name='azure.core').setLevel(logging.WARNING)
+
+DEFAULT_SCOPES = ["https://graph.microsoft.com/.default"]
+
+
+class SharePointTrigger(HTTPHook):
+    def __init__(
+        self,
+        *args,
+        webhook_url: str,
+        tenant_id: Optional[str] = None,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        resource: Optional[str] = None,
+        **kwargs
+    ):
+        self.methods: list = ['POST']
+        super().__init__(*args, **kwargs)
+        self.tenant_id = tenant_id or SHAREPOINT_TENANT_ID
+        self._tenant_name = kwargs.get('tenant', SHAREPOINT_TENANT_NAME)
+        self.client_id = client_id or SHAREPOINT_APP_ID
+        self.client_secret = client_secret or SHAREPOINT_APP_SECRET
+        self.site_name: str = kwargs.get('site_name')
+        self._host: str = kwargs.get('host', SHAREPOINT_DEFAULT_HOST)
+        self.folder_path: str = kwargs.get('folder_path')
+        self.site_id: Optional[str] = None
+        self.resource = resource
+        self.webhook_url = webhook_url
+        self.client_state: str = kwargs.get('client_state', 'flowtask_state')
+        self.changetype: str = kwargs.get('changetype', 'updated')
+        self.access_token = None
+        self.subscription_id = None
+        self._msapp: Callable = None
+        self._graph_client: Callable = None
+        self.renewal_task = None
+        self.renewal_interval = 3600 * 24  # Renew every 24 hours
+        self.scopes = kwargs.get('scopes', DEFAULT_SCOPES)
+
+    def get_client(self):
+        return ClientSecretCredential(
+            tenant_id=self.tenant_id,
+            client_id=self.client_id,
+            client_secret=self.client_secret
+        )
+
+    def get_graph_client(
+        self,
+        client: Any,
+        scopes: Optional[list] = None
+    ):
+        if not scopes:
+            scopes = self.scopes
+        return GraphServiceClient(credentials=client, scopes=scopes)
+
+    async def authenticate(self):
+        """
+        Authenticates the client with Azure AD and initializes the Graph client.
+
+        This method creates a ClientSecretCredential using the tenant ID, client ID,
+        and client secret. It then sets up an AzureIdentityAuthenticationProvider,
+        initializes a GraphRequestAdapter, and finally creates a GraphServiceClient.
+
+        The method doesn't take any parameters as it uses the instance variables
+        for authentication details.
+
+        Returns:
+            None
+        """
+        self._client = ClientSecretCredential(
+            tenant_id=self.tenant_id,
+            client_id=self.client_id,
+            client_secret=self.client_secret
+        )
+        auth_provider = AzureIdentityAuthenticationProvider(self._client)
+        self._request_adapter = GraphRequestAdapter(auth_provider)
+        self._graph_client = GraphServiceClient(
+            credentials=self._client,
+            scopes=self.scopes,
+            request_adapter=self._request_adapter
+        )
+
+    async def create_subscription(self):
+        if not self._graph_client:
+            await self.authenticate()
+        expiration_datetime = (datetime.datetime.utcnow() + datetime.timedelta(minutes=4230)).isoformat() + 'Z'
+        subscription_data = Subscription(
+            change_type=self.changetype,
+            notification_url=self.webhook_url,
+            resource=self.resource,
+            expiration_date_time=expiration_datetime,
+            client_state=self.client_state,
+        )
+        # subscription_data.include_resource_data = True
+        self._logger.info(
+            f"Using webhook URL: {self.webhook_url}"
+        )
+        self._logger.info(f"Subscription data: {subscription_data}")
+        response = await self._graph_client.subscriptions.post(subscription_data)
+        if response:
+            self.subscription_id = response.id
+        else:
+            self._logger.error("Failed to create subscription")
+            raise Exception(
+                "Failed to create Sharepoint subscription"
+            )
+
+    async def delete_subscription(self):
+        if not self.subscription_id:
+            return
+        if not self._graph_client:
+            await self.authenticate()
+        result = await self._graph_client.subscriptions.by_subscription_id(
+            self.subscription_id
+        ).delete()
+        if result is None:
+            self._logger.info("Subscription deleted")
+        else:
+            self._logger.error("Failed to delete subscription")
+
+    async def renew_subscription(self):
+        while True:
+            await asyncio.sleep(self.renewal_interval)
+            if not self.subscription_id:
+                continue
+            if not self._graph_client:
+                await self.authenticate()
+            expiration_datetime = (datetime.datetime.utcnow() + datetime.timedelta(minutes=4230)).isoformat() + 'Z'
+            subscription = Subscription()
+            subscription.expiration_date_time = expiration_datetime
+            response = await self._graph_client.subscriptions.by_subscription_id(
+                self.subscription_id
+            ).patch(subscription)
+            if response:
+                self._logger.info("Subscription renewed")
+            else:
+                self._logger.error("Failed to renew subscription")
+
+    async def on_startup(self, app):
+        self._logger.info("Starting SharePointTrigger")
+        asyncio.create_task(self._subscription_creation())
+
+    async def _subscription_creation(self):
+        # delaying startup for 5 sconds
+        await asyncio.sleep(5)
+        if not self.resource:
+            await self.get_site_id()
+            self.build_resource()
+        await self.create_subscription()
+        self.renewal_task = asyncio.create_task(
+            self.renew_subscription()
+        )
+
+    async def on_shutdown(self, app):
+        self._logger.info("Shutting down SharePointTrigger")
+        if self.renewal_task:
+            self.renewal_task.cancel()
+        await self.delete_subscription()
+
+    async def post(self, request: web.Request):
+        self._logger.info("Received POST request")
+        # Handle validation token
+        validation_token = request.query.get('validationToken')
+        if validation_token:
+            self._logger.info(f"Received validation token: {validation_token}")
+            return web.Response(text=validation_token, status=200)
+
+        # Handle notifications
+        try:
+            data = await request.json()
+            self._logger.info(f"Received notification data: {data}")
+        except Exception as e:
+            self._logger.error(f"Failed to parse request JSON: {e}")
+            return web.Response(status=400)
+
+        # Verify clientState
+        if self.client_state:
+            client_state = data.get('value', [{}])[0].get('clientState')
+            if client_state != self.client_state:
+                self._logger.warning("Invalid clientState in notification")
+                return web.Response(status=202)
+
+        # Process notifications
+        await self.process_notifications(data)
+        return web.Response(status=202)
+
+    async def process_notifications(self, data):
+        notifications = data.get('value', [])
+        for notification in notifications:
+            resource = notification.get('resource')
+            self._logger.info(
+                f"Received notification for resource: {resource}"
+            )
+            # Fetch details about the changed file
+            file_details = await self.get_resource_details(resource)
+            # Run actions with the file details
+            await self.run_actions(file_details=file_details)
+
+    async def get_resource_details(self, resource):
+        client = self.get_client()
+        # Get the access token
+        token = await client.get_token("https://graph.microsoft.com/.default")
+        access_token = token.token
+
+        # Build the full URL
+        resource_url = f"https://graph.microsoft.com/v1.0/{resource.lstrip('/')}"
+
+        # Log the resource URL for debugging
+        self._logger.info(f"Fetching resource details from URL: {resource_url}")
+
+        headers = {
+            'Authorization': f'Bearer {access_token}',
+            'Accept': 'application/json'
+        }
+
+        async with aiohttp.ClientSession() as session:
+            async with session.get(resource_url, headers=headers) as response:
+                if response.status == 200:
+                    data = await response.json()
+                    return data
+                else:
+                    error_text = await response.text()
+                    self._logger.error(f"Failed to get resource details: {error_text}")
+                    return {}
+
+    async def get_site_id(self):
+        if not self._graph_client:
+            await self.authenticate()
+        # Ensure site_name does not start with a slash
+        site_name = self.site_name.lstrip('/')
+
+        # Construct the site URL
+        site_request_url = f"https://graph.microsoft.com/v1.0/sites/{self._tenant_name}.sharepoint.com:/sites/{site_name}"  # noqa
+
+        # Log the constructed URL for debugging
+        self._logger.info(
+            f"Site request URL: {site_request_url}"
+        )
+
+        # Use `with_url` with the relative path, not the full URL
+        site_request_builder = self._graph_client.sites.with_url(site_request_url)
+
+        # Fetch the site information
+        site = await site_request_builder.get()
+        if site:
+            self.site_id = site.additional_data.get('id').split(',')[1]
+            self._logger.info(
+                f"Site ID obtained: {self.site_id}"
+            )
+        else:
+            self._logger.error(
+                f"Failed to get site ID for site: {self.site_name}"
+            )
+            raise Exception(
+                f"Failed to get site ID for site: {self.site_name}"
+            )
+
+    def build_resource(self):
+        if not self.site_id:
+            raise Exception("Site ID not available. Cannot build resource URL.")
+        folder_path = self.folder_path.strip('/')
+        encoded_folder_path = quote(folder_path)
+        # /{encoded_folder_path}:/children
+        self.resource = f"/sites/{self.site_id}/drive/root"
+        self._logger.info(
+            f"Resource URL built: {self.resource}"
+        )

flowtask/hooks/types/ssh.py

@@ -0,0 +1,141 @@
+import time
+import fnmatch
+import threading
+import paramiko
+from navconfig.logging import logging
+from .watch import BaseWatchdog, BaseWatcher
+from ...interfaces import CacheSupport
+
+logging.getLogger("paramiko").setLevel(logging.WARNING)
+
+class SFTPWatcher(BaseWatcher):
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        username: str,
+        password: str,
+        path: str,
+        interval: int = 300,  # Intervalo de verificación en segundos
+        max_retries: int = 5,  # Número máximo de reintentos
+        retry_delay: int = 60,  # Tiempo de espera entre reintentos en segundos
+        **kwargs,
+    ):
+        super(SFTPWatcher, self).__init__(**kwargs)
+        self.host = host
+        self.port = port
+        self.user = username
+        self.password = password
+        self.interval = interval
+        self.path = path
+        self._expiration = kwargs.pop("every", None)
+        self.max_retries = max_retries
+        self.retry_delay = retry_delay
+        self.stop_event = kwargs.get("stop_event", threading.Event())  # Evento para detener el Watchdog
+
+    def close_watcher(self):
+        pass
+
+    def run(self, *args, **kwargs):
+        retries = 0
+        while not self.stop_event.is_set():
+            try:
+                self._logger.info(f"Intentando conectar a {self.host}:{self.port}")
+                # Conectar al servidor SSH
+                ssh_client = paramiko.SSHClient()
+                ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+                ssh_client.connect(
+                    hostname=self.host,
+                    port=self.port,
+                    username=self.user,
+                    password=self.password,
+                    timeout=10  # Timeout de 10 segundos
+                )
+                # Conectar al servidor SFTP
+                sftp_client = ssh_client.open_sftp()
+                self._logger.info(f"Conexión SFTP establecida en {self.host}:{self.port}")
+
+                # Verificar si el archivo o directorio existe
+                try:
+                    directory, pattern = self.path.rsplit("/", 1)
+                    files = sftp_client.listdir(directory)
+                    matching_files = fnmatch.filter(files, pattern)
+
+                    found_files = []
+                    with CacheSupport(every=self._expiration) as cache:
+                        for file in matching_files:
+                            filepath = f"{directory}/{file}"
+                            if cache.exists(filepath):
+                                continue
+                            stat = sftp_client.stat(filepath)
+                            file_info = {
+                                "filename": file,
+                                "directory": directory,
+                                "path": filepath,
+                                "host": self.host,
+                                "size": stat.st_size,
+                                "perms": oct(stat.st_mode),
+                                "modified": time.ctime(stat.st_mtime),
+                            }
+                            self._logger.notice(f"Encontrado {self.path} en {self.host}: {file_info}")
+                            found_files.append(file_info)
+                            cache.setexp(filepath, value=filepath)
+                        if found_files:
+                            args = {"files": found_files, **kwargs}
+                            self.parent.call_actions(**args)
+                except FileNotFoundError:
+                    self._logger.warning(f"Ruta no encontrada: {self.path}")
+                finally:
+                    sftp_client.close()
+                    ssh_client.close()
+                    self._logger.info(f"Desconectado de {self.host}:{self.port}")
+                    retries = 0  # Reiniciar el contador de reintentos tras una conexión exitosa
+            except (paramiko.SSHException, paramiko.AuthenticationException, paramiko.BadHostKeyException) as e:
+                self._logger.error(f"Error de conexión SSH/SFTP: {e}")
+                retries += 1
+                if retries >= self.max_retries:
+                    self._logger.error(f"Se alcanzó el máximo de reintentos ({self.max_retries}). Deteniendo Watchdog.")
+                    break
+                self._logger.info(f"Reintentando en {self.retry_delay} segundos... (Intento {retries}/{self.max_retries})")
+                self._sleep_with_stop(self.retry_delay)
+                continue
+            except Exception as e:
+                self._logger.error(f"Se produjo un error al verificar el servidor: {e}")
+                retries += 1
+                if retries >= self.max_retries:
+                    self._logger.error(f"Se alcanzó el máximo de reintentos ({self.max_retries}). Deteniendo Watchdog.")
+                    break
+                self._logger.info(f"Reintentando en {self.retry_delay} segundos... (Intento {retries}/{self.max_retries})")
+                self._sleep_with_stop(self.retry_delay)
+                continue
+
+            # Esperar el intervalo antes de la siguiente verificación
+            self._sleep_with_stop(self.interval)
+
+    def _sleep_with_stop(self, duration: int):
+        """
+        Duerme de manera interrumpible, verificando el evento de parada cada segundo.
+        """
+        for _ in range(duration):
+            if self.stop_event and self.stop_event.is_set():
+                break
+            time.sleep(1)
+
+
+class SFTPWatchdog(BaseWatchdog):
+    def create_watcher(self, *args, **kwargs) -> BaseWatcher:
+        credentials = kwargs.pop("credentials", {})
+        interval = kwargs.pop("interval", 300)
+        self.mask_start(**kwargs)
+        self.credentials = self.set_credentials(credentials)
+        self.path = self.mask_replacement(kwargs.pop("path", None))
+        # Crear un evento de parada para controlar la detención del Watchdog
+        stop_event = threading.Event()
+        self.stop_event = stop_event
+        return SFTPWatcher(
+            **self.credentials,
+            path=self.path,
+            interval=interval,
+            stop_event=self.stop_event,
+            **kwargs
+        )

flowtask/hooks/types/tagged.py

@@ -0,0 +1,59 @@
+from .watch import BaseWatcher
+from .imap import IMAPWatchdog, ImapWatcher
+
+
+class TaggedImapWatcher(ImapWatcher):
+    def __init__(self, tagged_address: str, *args, **kwargs):
+        self.tagged_address = tagged_address
+        super(TaggedImapWatcher, self).__init__(*args, **kwargs)
+
+    def process_email(self, email_id, **kwargs):
+        # Explicitly mark the email as SEEN
+        self.imap_server.store(email_id, "+FLAGS", "\SEEN")
+        typ, _ = self.imap_server.fetch(email_id, "(FLAGS)")
+        if typ != "OK":
+            self._logger.warning(f"Cannot mark email {email_id} as read (SEEN)")
+        super(TaggedImapWatcher, self).process_email(email_id, **kwargs)
+
+    def build_search_criteria(self, search_criteria):
+        criteria = []
+        for key, value in search_criteria.items():
+            if value is None:
+                criteria.append(key)
+            else:
+                criteria.append(f'({key} "{value}")')
+        criteria.append(f'(TO "{self.tagged_address}") UNSEEN')
+        return " ".join(criteria)
+
+
+class TaggedIMAPWatchdog(IMAPWatchdog):
+    def get_tagged_address(self, username: str, tag: str) -> str:
+        user, domain = username.split("@")
+        return f"{user}+{tag}@{domain}"
+
+    def create_watcher(self, *args, **kwargs) -> BaseWatcher:
+        tag = kwargs.pop("tag", None)
+        if not tag:
+            raise ValueError("TAG is required for TaggedIMAPWatchdog")
+        credentials = kwargs.pop("credentials", {})
+        self.mailbox = kwargs.pop("mailbox", "INBOX")
+        interval = kwargs.pop("interval", 10)
+        authmech = credentials.pop("authmech", None)
+        self.mask_start(**kwargs)
+        search = self.processing_search(kwargs.pop("search", {}))
+        # TODO: processing with masks the Search criteria:
+        self.credentials = self.set_credentials(credentials)
+        alias_address = kwargs.pop("alias_address", None)
+        if alias_address:
+            tagged_address = alias_address
+        else:
+            tagged_address = self.get_tagged_address(credentials["user"], tag)
+        return TaggedImapWatcher(
+            **self.credentials,
+            mailbox=self.mailbox,
+            interval=interval,
+            authmech=authmech,
+            search=search,
+            tagged_address=tagged_address,
+            **kwargs,
+        )

flowtask/hooks/types/upload.py

@@ -0,0 +1,85 @@
+import os
+from typing import Optional, List
+from aiohttp import web
+from .http import HTTPHook
+
+
+class UploadHook(HTTPHook):
+    """UploadHook.
+
+    A Trigger that handles file uploads via HTTP POST/PUT requests.
+    """
+
+    methods: list = ["POST", "PUT"]
+
+    def __init__(
+        self,
+        *args,
+        allowed_mime_types: Optional[List[str]] = None,
+        allowed_file_names: Optional[List[str]] = None,
+        **kwargs
+    ):
+        super().__init__(*args, **kwargs)
+        self.allowed_mime_types = allowed_mime_types
+        self.allowed_file_names = allowed_file_names
+
+    async def post(self, request: web.Request):
+        # Use handle_upload to get uploaded files and form data
+        try:
+            uploaded_files_info, form_data = await self.handle_upload(request)
+        except Exception as e:
+            # Return error response
+            return self.response(
+                response={'error': str(e)},
+                status=400,
+                content_type='application/json'
+            )
+
+        # Validate files if necessary
+        for file_info in uploaded_files_info:
+            file_path = file_info['file_path']
+            file_name = file_info['file_name']
+            mime_type = file_info['mime_type']
+
+            if self.allowed_mime_types and mime_type not in self.allowed_mime_types:
+                return self.response(
+                    response={'error': f'Invalid mime type: {mime_type}'},
+                    status=400,
+                    content_type='application/json'
+                )
+
+            if self.allowed_file_names and file_name not in self.allowed_file_names:
+                return self.response(
+                    response={'error': f'Invalid file name: {file_name}'},
+                    status=400,
+                    content_type='application/json'
+                )
+
+        # Prepare data to pass to run_actions
+        data = {
+            'uploaded_files': [str(file_info['file_path']) for file_info in uploaded_files_info],
+            'form_data': form_data
+        }
+        # Run actions
+        result = await self.run_actions(**data)
+        return self.response(
+            response=result,
+            status=self.default_status
+        )
+
+    async def put(self, request: web.Request):
+        return await self.post(request)
+
+    async def run_actions(self, **data):
+        result = await super().run_actions(**data)
+        try:
+            # Clean up uploaded files after actions have run
+            uploaded_files = data.get('uploaded_files', [])
+            for file_path in uploaded_files:
+                try:
+                    os.remove(file_path)
+                    self._logger.info(f"Deleted temporary file: {file_path}")
+                except Exception as e:
+                    self._logger.warning(f"Failed to delete temporary file {file_path}: {e}")
+        finally:
+            return result