flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/hooks/types/imap.py

@@ -0,0 +1,200 @@
+from typing import Union, Optional
+import imaplib
+import time
+from functools import partial
+import ssl
+from ...exceptions import ComponentError
+from ...interfaces.azureauth import AzureAuth
+from .watch import BaseWatchdog, BaseWatcher
+from ...interfaces import CacheSupport
+
+
+class ImapWatcher(BaseWatcher):
+    def __init__(
+        self,
+        host: str,
+        port: int,
+        user: str,
+        password: str,
+        mailbox="INBOX",
+        use_ssl: bool = True,
+        interval: int = 320,
+        authmech: str = None,
+        search: Optional[Union[str, dict, list]] = None,
+        **kwargs,
+    ):
+        super(ImapWatcher, self).__init__(**kwargs)
+        self.host = host
+        self.port = port
+        self.user = user
+        self.password = password
+        self.mailbox = mailbox
+        self.interval = interval
+        self.authmech = authmech
+        self.use_ssl = use_ssl
+        self.search = search
+        self.args = kwargs
+        self._expiration = kwargs.pop("expiration", None)
+
+    def close_watcher(self):
+        try:
+            self.imap_server.logout()
+        except OSError as exc:
+            self._logger.error(f"Error closing IMAP Connection: {exc}")
+
+    def connect(self, max_retries: int = 3, delay: int = 10):
+        if self.use_ssl is True:
+            sslcontext = ssl.create_default_context()
+            server = partial(
+                imaplib.IMAP4_SSL,
+                self.host,
+                self.port,
+                timeout=10,
+                ssl_context=sslcontext,
+            )
+        else:
+            server = partial(imaplib.IMAP4, self.host, self.port, timeout=10)
+        # azure auth:
+        if isinstance(server, str):
+            raise ComponentError(
+                f"Failed to configure to IMAP Server: {server}"
+            )
+        azure = AzureAuth()  # default values
+        for attempt in range(max_retries):
+            try:
+                self.imap_server = server()
+                if self.authmech == "XOAUTH2":
+                    self._logger.debug("IMAP Auth XOAUTH2")
+                    try:
+                        result, msg = self.imap_server.authenticate(
+                            self.authmech,
+                            lambda x: azure.binary_token(self.user, self.password),
+                        )
+                        if result != "OK":
+                            raise ComponentError(
+                                f"IMAP: Wrong response: {result} message={msg}"
+                            )
+                    except AttributeError as err:
+                        raise ComponentError(
+                            f"Login Forbidden, wrong username or password: {err}"
+                        ) from err
+                else:
+                    self._logger.debug("IMAP Basic Login")
+                    ## making the server login:
+                    r = self.imap_server.login(self.user, self.password)
+                    if r.result == "NO":
+                        raise ComponentError(
+                            f"Login Forbidden, Server Disconnected: {r}"
+                        )
+                ### Select Mailbox
+                self.imap_server.select(self.mailbox, readonly=False)
+                # If connection is successful, break out of the loop
+                break
+            except Exception as exc:
+                server = f"{self.host}:{self.port}"
+                if attempt == max_retries - 1:
+                    raise ComponentError(
+                        f"Could not connect to IMAP Server {self.host}:{self.port}: {exc}"
+                    ) from exc
+                # Otherwise, log the exception and wait before retrying
+                self._logger.warning(
+                    f"Attempt {attempt + 1} to connect to IMAP {server}: {exc}. Retrying in {delay} sec."
+                )
+                time.sleep(delay)
+
+    def build_search_criteria(self, search_criteria):
+        criteria = []
+        for key, value in search_criteria.items():
+            if value is None:
+                criteria.append(key)
+            else:
+                criteria.append(f'({key} "{value}")')
+        return " ".join(criteria)
+
+    def process_email(self, email_id, **kwargs):
+        self._logger.debug(f"Email {email_id} has processed.")
+        self.parent.call_actions(**kwargs)
+
+    def run(self, **kwargs):
+        try:
+            self.connect()
+        except ComponentError as exc:
+            self._logger.error(
+                f"Error connecting to IMAP server: {exc}"
+            )
+            return
+        while not self.stop_event.is_set():
+            try:
+                search_criteria = self.build_search_criteria(self.search)
+                self._logger.debug(f"IMAP: Running Search: {search_criteria}")
+                result, data = self.imap_server.search(None, search_criteria)
+                if result == "OK":
+                    # TODO:
+                    emails = data[0].split()
+                    unseen_emails = len(emails)
+                    self._logger.notice(
+                        f"Found {unseen_emails} emails in {self.mailbox}"
+                    )
+                    for email_id in emails:
+                        # Fetch the email's structure
+                        status, mail_data = self.imap_server.fetch(
+                            email_id, "(BODYSTRUCTURE)"
+                        )
+                        if status == "OK":
+                            with CacheSupport(every=self._expiration) as cache:
+                                if cache.exists(email_id):
+                                    continue
+                                # check if email has attachments:
+                                if (
+                                    "has_attachments" in self.args and self.args["has_attachments"] is True
+                                ):
+                                    structure = mail_data[0]
+                                    if isinstance(structure, bytes):
+                                        structure = structure.decode("utf-8")
+                                    # Check if the structure contains a filename parameter
+                                    if '("attachment"' in structure:
+                                        cache.setexp(email_id, value=structure)
+                                        self.process_email(email_id, **kwargs)
+                                else:
+                                    structure = mail_data[0]
+                                    cache.setexp(email_id, value=structure)
+                                    self.process_email(email_id, **kwargs)
+            except Exception as e:
+                print(f"An error occurred while checking the mailbox: {e}")
+                # Reconnect if an error occurs
+                self.connect()
+            # Wait for the interval, but allow the sleep to be interrupted by the signal
+            try:
+                for _ in range(self.interval):
+                    if self.stop_event.is_set():
+                        break
+                    time.sleep(1)
+            except KeyboardInterrupt:
+                break
+
+
+class IMAPWatchdog(BaseWatchdog):
+    def processing_search(self, search_terms: dict):
+        search = {}
+        for key, value in search_terms.items():
+            val = self.mask_replacement(value)
+            search[key] = val
+        return search
+
+    def create_watcher(self, *args, **kwargs) -> BaseWatcher:
+        credentials = kwargs.pop("credentials", {})
+        self.mailbox = kwargs.pop("mailbox", "INBOX")
+        interval = kwargs.pop("interval", 60)
+        authmech = credentials.pop("authmech", None)
+        self.mask_start(**kwargs)
+        search = self.processing_search(kwargs.pop("search", {}))
+        # TODO: processing with masks the Search criteria:
+        self.credentials = self.set_credentials(credentials)
+        return ImapWatcher(
+            **self.credentials,
+            mailbox=self.mailbox,
+            interval=interval,
+            authmech=authmech,
+            search=search,
+            **kwargs,
+        )

flowtask/hooks/types/jira.py

@@ -0,0 +1,279 @@
+from typing import Optional, Any, List, Dict
+from dataclasses import dataclass, field
+from aiohttp import web
+import hmac
+import hashlib
+from datamodel import BaseModel
+from datamodel.libs.mapping import ClassDict
+from datamodel.parsers.json import json_encoder, json_decoder
+from ...conf import (
+    JIRA_SECRET_TOKEN,
+    JIRA_API_TOKEN,
+    JIRA_USERNAME,
+    JIRA_INSTANCE,
+    JIRA_PROJECT,
+)
+from .http import HTTPHook
+
+
+@dataclass
+class AvatarUrls:
+    _48x48: str = field(metadata={'alias': '48x48'})
+    _24x24: str = field(metadata={'alias': '24x24'})
+    _16x16: str = field(metadata={'alias': '16x16'})
+    _32x32: str = field(metadata={'alias': '32x32'})
+
+@dataclass
+class User:
+    self: str
+    accountId: str
+    avatarUrls: AvatarUrls
+    displayName: str
+    active: bool
+    timeZone: str
+    accountType: str
+
+@dataclass
+class StatusCategory:
+    self: str
+    id: int
+    key: str
+    colorName: str
+    name: str
+
+@dataclass
+class Status:
+    self: str
+    description: str
+    iconUrl: str
+    name: str
+    id: str
+    statusCategory: StatusCategory
+
+@dataclass
+class Priority:
+    self: str
+    iconUrl: str
+    name: str
+    id: str
+
+@dataclass
+class Progress:
+    progress: int
+    total: int
+
+@dataclass
+class AggregateProgress:
+    progress: int
+    total: int
+
+@dataclass
+class Votes:
+    self: str
+    votes: int
+    hasVoted: bool
+
+@dataclass
+class Watches:
+    self: str
+    watchCount: int
+    isWatching: bool
+
+@dataclass
+class ProjectCategory:
+    self: str
+    id: str
+    description: str
+    name: str
+
+@dataclass
+class Project:
+    self: str
+    id: str
+    key: str
+    name: str
+    projectTypeKey: str
+    simplified: bool
+    avatarUrls: AvatarUrls
+    projectCategory: ProjectCategory
+
+@dataclass
+class IssueType:
+    self: str
+    id: str
+    description: str
+    iconUrl: str
+    name: str
+    subtask: bool
+    avatarId: int
+    hierarchyLevel: int
+
+@dataclass
+class TimeTracking:
+    originalEstimate: Optional[str] = None
+    remainingEstimate: Optional[str] = None
+    timeSpent: Optional[str] = None
+
+class JiraIssue(BaseModel):
+    """JiraIssue.
+
+    A BaseModel to represent a Jira Issue.
+    """
+    self: str
+    id: str
+    key: str
+    event_type: str
+    timestamp: int
+    webhook_event: str
+    issue_event_type_name: str
+    issue_status: Optional[str]
+    description: Optional[str]
+    summary: str
+    changelog: Optional[Dict]
+    user: Optional[User]
+    issue: dict
+    fields: Optional[Dict]
+    priority: Optional[Priority]
+    status: Optional[Status]
+    creator: Optional[User]
+    reporter: Optional[User]
+    aggregateprogress: AggregateProgress
+    progress: Progress
+    votes: Votes
+    issuetype: IssueType
+    project: Project
+    watches: Watches
+    timetracking: TimeTracking
+
+    class Meta:
+        strict: bool = False
+        frozen: bool = False
+
+class JiraTrigger(HTTPHook):
+    """JiraTrigger.
+
+    A Trigger that handles Jira webhooks for issue events.
+    """
+
+    def __init__(
+        self,
+        *args,
+        secret_token: str = None,
+        **kwargs
+    ):
+        super().__init__(*args, **kwargs)
+        self.secret_token = secret_token or JIRA_SECRET_TOKEN
+        self.username = kwargs.get('username', JIRA_USERNAME)
+        self.instance = kwargs.get('instance', JIRA_INSTANCE)
+        self.project = kwargs.get('project', JIRA_PROJECT)
+        self.url = kwargs.get('url', '/api/v1/webhook/jira/')
+        self.methods = ['POST']
+
+    async def post(self, request: web.Request):
+        try:
+            # Verify the request
+            if self.secret_token:
+                signature = request.headers.get('X-Hub-Signature')
+                if not signature:
+                    return web.Response(status=401, text='Unauthorized')
+                body = await request.read()
+                computed_signature = 'sha256=' + hmac.new(
+                    self.secret_token.encode('utf-8'),
+                    body,
+                    hashlib.sha256
+                ).hexdigest()
+                if not hmac.compare_digest(signature, computed_signature):
+                    return web.Response(status=401, text='Unauthorized')
+                payload = json_decoder(body)
+            else:
+                payload = await request.json()
+            # Extract event details
+            webhook_event = payload.get('webhookEvent')
+            timestamp = payload.get('timestamp')
+            user = payload.get('user', {})
+            issue_type = payload.get('issue_event_type_name', 'issue_created')
+            changelog = payload.get('changelog', {})
+            issue = payload.get('issue', {})
+            issue_key = issue.get('key')
+            issue_fields = issue.get('fields', {})
+            issue_status = issue_fields.get('status', {}).get('name')
+
+            # Determine the event type
+            event_type = None
+            if webhook_event == 'jira:issue_created':
+                event_type = 'created'
+            elif webhook_event == 'jira:issue_updated':
+                # Check if the issue was closed
+                changelog = payload.get('changelog', {})
+                items = changelog.get('items', [])
+                for item in items:
+                    if item.get('field') == 'status':
+                        from_status = item.get('fromString')
+                        to_status = item.get('toString')
+                        if to_status.lower() == 'closed':
+                            event_type = 'closed'
+                        else:
+                            event_type = 'updated'
+                        break
+                else:
+                    event_type = 'updated'
+            elif webhook_event == 'jira:issue_deleted':
+                event_type = 'deleted'
+            if event_type:
+                # extracting information from issue:
+                aggregateprogress = issue_fields.get('aggregateprogress', {})
+                issuetype = issue_fields.get('issuetype', {})
+                creator = issue_fields.get('creator', {})
+                reporter = issue_fields.get('reporter', {})
+                project = issue_fields.get('project', {})
+                watches = issue_fields.get('watches', {})
+                timetracking = issue_fields.get('timetracking', {})
+                priority = issue_fields.get('priority', {})
+                # Create the JiraIssue object
+                jira_issue = JiraIssue(
+                    event_type=event_type,
+                    issue_status=issue_status,
+                    timestamp=timestamp,
+                    webhook_event=webhook_event,
+                    issue_event_type_name=issue_type,
+                    aggregateprogress=aggregateprogress,
+                    issuetype=issuetype,
+                    creator=creator,
+                    reporter=reporter,
+                    description=issue_fields.get('description'),
+                    summary=issue_fields.get('summary'),
+                    changelog=changelog,
+                    user=user,
+                    priority=priority,
+                    project=project,
+                    watches=watches,
+                    timetracking=timetracking,
+                    **issue
+                )
+                # Prepare data to pass to actions
+                data = {
+                    'webhook_event': webhook_event,
+                    'event_type': event_type,
+                    'issue_key': issue_key,
+                    'issue_fields': issue_fields,
+                    'issue': jira_issue
+                }
+                # Run actions
+                result = await self.run_actions(**data)
+                return self.response(
+                    response=result,
+                    status=self.default_status
+                )
+            else:
+                return web.Response(
+                    status=200,
+                    text='Event ignored'
+                )
+        except Exception as e:
+            self._logger.error(
+                f"Error processing Jira webhook: {e}"
+            )
+            return web.Response(
+                status=500,
+                text='Jira: Internal Server Error'
+            )

flowtask/hooks/types/mail.py

@@ -0,0 +1,205 @@
+import asyncio
+import datetime
+from typing import Any, Optional
+from aiohttp import web
+from azure.identity.aio import ClientSecretCredential
+from msgraph import GraphRequestAdapter, GraphServiceClient
+from msgraph.generated.models.subscription import Subscription
+from msgraph.generated.models.message import Message
+from kiota_authentication_azure.azure_identity_authentication_provider import AzureIdentityAuthenticationProvider
+from navconfig.logging import logging
+from .http import HTTPHook
+from ...conf import (
+    SHAREPOINT_APP_ID,
+    SHAREPOINT_APP_SECRET,
+    SHAREPOINT_TENANT_ID,
+    SHAREPOINT_TENANT_NAME
+)
+
+logging.getLogger(name='azure.identity.aio').setLevel(logging.WARNING)
+logging.getLogger(name='azure.core').setLevel(logging.WARNING)
+
+DEFAULT_SCOPES = ["https://graph.microsoft.com/.default"]
+
+
+class EmailTrigger(HTTPHook):
+
+    def __init__(
+        self,
+        *args,
+        webhook_url: str,
+        tenant_id: Optional[str] = None,
+        client_id: Optional[str] = None,
+        client_secret: Optional[str] = None,
+        resource: Optional[str] = None,
+        **kwargs
+    ):
+        super().__init__(*args, **kwargs)
+        self.tenant_id = tenant_id or SHAREPOINT_TENANT_ID
+        self._tenant_name = kwargs.get('tenant', SHAREPOINT_TENANT_NAME)
+        self.client_id = client_id or SHAREPOINT_APP_ID
+        self.client_secret = client_secret or SHAREPOINT_APP_SECRET
+        self.webhook_url = webhook_url
+        self.client_state: str = kwargs.get('client_state', 'flowtask_state')
+        self.changetype: str = kwargs.get('changetype', 'created')
+        self.subscription_id = None
+        self._graph_client: GraphServiceClient = None
+        self._request_adapter: GraphRequestAdapter = None
+        self.renewal_task = None
+        self.renewal_interval = 3600 * 23  # Renew every 23 hours
+
+    def get_client(self):
+        return ClientSecretCredential(
+            tenant_id=self.tenant_id,
+            client_id=self.client_id,
+            client_secret=self.client_secret
+        )
+
+    def get_graph_client(
+        self,
+        client: Any,
+        scopes: Optional[list] = None
+    ):
+        if not scopes:
+            scopes = self.scopes
+        return GraphServiceClient(credentials=client, scopes=scopes)
+
+    async def authenticate(self):
+        """
+        Authenticates the client with Azure AD and initializes the Graph client.
+
+        This method creates a ClientSecretCredential using the tenant ID, client ID,
+        and client secret. It then sets up an AzureIdentityAuthenticationProvider,
+        initializes a GraphRequestAdapter, and finally creates a GraphServiceClient.
+
+        The method doesn't take any parameters as it uses the instance variables
+        for authentication details.
+
+        Returns:
+            None
+        """
+        self._client = self.get_client()
+        auth_provider = AzureIdentityAuthenticationProvider(self._client)
+        self._request_adapter = GraphRequestAdapter(auth_provider)
+        self._graph_client = GraphServiceClient(
+            credentials=self._client,
+            scopes=self.scopes,
+            request_adapter=self._request_adapter
+        )
+
+    async def create_subscription(self):
+        if not self._graph_client:
+            await self.authenticate()
+
+        expiration_datetime = (datetime.datetime.utcnow() + datetime.timedelta(hours=23)).isoformat() + 'Z'
+        subscription = Subscription(
+            change_type=self.changetype,
+            notification_url=self.webhook_url,
+            resource="me/mailFolders('Inbox')/messages",
+            expiration_date_time=expiration_datetime,
+            client_state=self.client_state,
+            latest_supported_tls_version="v1_2",
+        )
+
+        result = await self._graph_client.subscriptions.post(body=subscription)
+        if result:
+            self.subscription_id = result.id
+            self._logger.info(f"Subscription created with ID: {self.subscription_id}")
+
+        else:
+            self._logger.error("Failed to create subscription")
+            raise Exception("Failed to create email subscription")
+
+    async def delete_subscription(self):
+        if not self.subscription_id:
+            return
+        if not self._graph_client:
+            await self.authenticate()
+
+        await self._graph_client.subscriptions.by_subscription_id(self.subscription_id).delete()
+        self._logger.info("Subscription deleted")
+
+    async def renew_subscription(self):
+        while True:
+            await asyncio.sleep(self.renewal_interval)
+            if not self.subscription_id:
+                continue
+            if not self._graph_client:
+                await self.authenticate()
+
+            expiration_datetime = (datetime.datetime.utcnow() + datetime.timedelta(hours=23)).isoformat() + 'Z'
+            subscription = Subscription(
+                expiration_date_time=expiration_datetime
+            )
+            await self._graph_client.subscriptions.by_subscription_id(self.subscription_id).patch(body=subscription)
+            self._logger.info("Subscription renewed")
+
+    async def post(self, request: web.Request):
+        self._logger.info("Received POST request")
+        # Handle validation token
+        validation_token = request.query.get('validationToken')
+        if validation_token:
+            self._logger.info(f"Received validation token: {validation_token}")
+            return web.Response(text=validation_token, status=200)
+
+        # Handle notifications
+        try:
+            data = await request.json()
+            self._logger.info(f"Received notification data: {data}")
+        except Exception as e:
+            self._logger.error(f"Failed to parse request JSON: {e}")
+            return web.Response(status=400)
+
+        # Verify clientState
+        if self.client_state:
+            client_state = data.get('value', [{}])[0].get('clientState')
+            if client_state != self.client_state:
+                self._logger.warning("Invalid clientState in notification")
+                return web.Response(status=202)
+
+        # Process notifications
+        await self.process_notifications(data)
+        return web.Response(status=202)
+
+    async def process_notifications(self, data):
+        notifications = data.get('value', [])
+        for notification in notifications:
+            resource = notification.get('resource')
+            self._logger.info(f"Received notification for resource: {resource}")
+            message_details = await self.get_message_details(resource)
+            # Run actions with the message details
+            await self.run_actions(message_details=message_details)
+
+    async def get_message_details(self, resource):
+        if not self._graph_client:
+            await self.authenticate()
+
+        # The resource should be something like "me/messages/{message-id}"
+        # Extract the message ID
+        parts = resource.split('/')
+        if 'messages' in parts:
+            message_index = parts.index('messages')
+            message_id = parts[message_index + 1]
+
+            message = await self._graph_client.me.messages.by_message_id(message_id).get()
+            return message.serialize()
+        else:
+            self._logger.error(
+                f"Cannot extract message ID from resource: {resource}"
+            )
+            return {}
+
+    async def on_startup(self, app):
+        self._logger.info("Starting EmailTrigger")
+        asyncio.create_task(self._subscription_creation())
+
+    async def _subscription_creation(self):
+        await asyncio.sleep(5)
+        await self.create_subscription()
+        self.renewal_task = asyncio.create_task(self.renew_subscription())
+
+    async def on_shutdown(self, app):
+        self._logger.info("Shutting down EmailTrigger")
+        if self.renewal_task:
+            self.renewal_task.cancel()
+        await self.delete_subscription()