flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/interfaces/FTPClient.py

@@ -0,0 +1,131 @@
+"""
+FTP Client.
+
+Overview
+
+This class provides functionalities for asynchronous operations with an FTP server, including establishing connections,
+checking directory existence, changing working directories, and downloading files.
+
+"""
+from typing import List, Dict
+from collections.abc import Callable
+import aioftp
+from ..exceptions import ComponentError
+from .client import ClientInterface
+
+
+"""
+FOR FTP over SSL
+
+from ftplib import FTP_TLS
+
+ftps = FTP_TLS(timeout=10)
+ftps.set_debuglevel(2)
+ftps.context.set_ciphers('DEFAULT@SECLEVEL=1')
+
+ftps.connect(host, port)
+
+ftps.login('user', 'password')
+# enable TLS
+ftps.auth()
+ftps.prot_p()
+ftps.retrlines('LIST')
+"""
+
+
+class FTPClient(ClientInterface):
+    block_size = 8192
+    algorithms: List = [
+        "ssh-rsa",
+        "ssh-dss",
+        "sk-ssh-ed25519-cert-v01@openssh.com",
+        "sk-ecdsa-sha2-nistp256-cert-v01@openssh.com",
+        "ssh-ed25519-cert-v01@openssh.com",
+        "ssh-ed448-cert-v01@openssh.com",
+        "ecdsa-sha2-nistp521-cert-v01@openssh.com",
+        "ecdsa-sha2-nistp384-cert-v01@openssh.com",
+        "ecdsa-sha2-nistp256-cert-v01@openssh.com",
+        "ecdsa-sha2-1.3.132.0.10-cert-v01@openssh.com",
+        "ssh-rsa-cert-v01@openssh.com",
+        "ssh-dss-cert-v01@openssh.com",
+        "sk-ssh-ed25519@openssh.com",
+        "sk-ecdsa-sha2-nistp256@openssh.com",
+        "ssh-ed25519",
+        "ssh-ed448",
+        "ecdsa-sha2-nistp521",
+        "ecdsa-sha2-nistp384",
+        "ecdsa-sha2-nistp256",
+        "ecdsa-sha2-1.3.132.0.10",
+        "rsa-sha2-256",
+        "rsa-sha2-512",
+    ]
+
+    async def close(self):
+        """Close Method."""
+        try:
+            await self._connection.quit()
+        except Exception as err:
+            print(f"Error on FTP disconnection, reason: {err!s}")
+
+    async def init_connection(
+        self, host, port, credentials: Dict, ssl: Callable = None
+    ):
+        """
+        init an FTP connection
+        """
+        args = {
+            "socket_timeout": 120,
+            "path_timeout": 30,
+            "encoding": "utf-8",
+            "ssl": ssl,
+        }
+        connection = None
+        try:
+            connection = aioftp.Client(**args)
+            await connection.connect(host, port)
+        except ValueError as err:
+            raise ComponentError(f"{err!s}") from err
+        except OSError as err:
+            raise ComponentError(f"FTP connection failed: {err!s}") from err
+        except Exception as err:
+            raise ComponentError(f"FTP Exception: {err!s}") from err
+        try:
+            await connection.login(*credentials.values())
+            return connection
+        except aioftp.StatusCodeError as err:
+            raise ComponentError(f"FTP connection Error: {err!s}") from err
+        except Exception as err:
+            raise ComponentError(f"FTP Exception: {err!s}") from err
+
+    def err_handler(self, err):
+        print(f"FTP Error: reason: {err.reason}, error: {err}")
+        return False
+
+    async def directory_exists(self, directory: str):
+        return await self._connection.exists(directory)
+
+    async def change_directory(self, directory: str):
+        await self._connection.change_directory(directory)
+
+    async def open(self):
+        pass
+
+    async def download_file(self, file: str, destination: str, rewrite: bool = False):
+        """download_file
+
+        Download a File from FTP based on Path.
+        Args:
+            file (str): file to be downloaded
+            destination (str): path to destination
+            rewrite (bool): file if exists, will be overwrite
+        TODO: Support for write_into and Renaming Files.
+        """
+        try:
+            await self._connection.download(
+                file,
+                destination=destination,
+                write_into=rewrite,
+                block_size=self.block_size,
+            )
+        except Exception as err:
+            raise ComponentError(f"FTP Download Exception: {err!s}") from err

flowtask/interfaces/GoogleCalendar.py

@@ -0,0 +1,201 @@
+from abc import ABC
+from typing import List, Dict
+import asyncio
+from datetime import datetime, timedelta
+import requests
+from googleapiclient.discovery import build
+from .GoogleClient import GoogleClient
+from ..exceptions import ComponentError
+
+
+class GoogleCalendarClient(GoogleClient, ABC):
+    """
+    Google Calendar Client for managing calendar events.
+    """
+
+    async def get_client(self):
+        """Get the Google Calendar client, with caching."""
+        if not hasattr(self, '_client'):
+            self.service = await asyncio.to_thread(build, 'calendar', 'v3', credentials=self.credentials)
+        return self.service
+
+    async def create_event(self, calendar_id: str, event: Dict) -> Dict:
+        """
+        Create an event in the specified Google Calendar.
+
+        Args:
+            calendar_id (str): The ID of the calendar to add the event to.
+            event (dict): The event details.
+
+        Returns:
+            dict: Details of the created event.
+        """
+        client = await self.get_client()
+        created_event = await asyncio.to_thread(client.events().insert(calendarId=calendar_id, body=event).execute)
+        print(f"Event created: {created_event.get('htmlLink')}")
+        return created_event
+
+    async def list_events(self, calendar_id: str, time_min: datetime, time_max: datetime, max_results: int = 10) -> List[Dict]:
+        """
+        List events in a specified time range.
+
+        Args:
+            calendar_id (str): The ID of the calendar to list events from.
+            time_min (datetime): Start time to retrieve events.
+            time_max (datetime): End time to retrieve events.
+            max_results (int): Maximum number of events to retrieve (default is 10).
+
+        Returns:
+            list: List of events within the time range.
+        """
+        client = await self.get_client()
+        events_result = await asyncio.to_thread(
+            client.events().list,
+            calendarId=calendar_id,
+            timeMin=time_min.isoformat() + 'Z',
+            timeMax=time_max.isoformat() + 'Z',
+            maxResults=max_results,
+            singleEvents=True,
+            orderBy="startTime"
+        )
+        events = events_result.execute().get('items', [])
+        return events
+
+    async def update_event(self, calendar_id: str, event_id: str, updated_event: Dict) -> Dict:
+        """
+        Update an existing event in the calendar.
+
+        Args:
+            calendar_id (str): The ID of the calendar containing the event.
+            event_id (str): The ID of the event to update.
+            updated_event (dict): The updated event details.
+
+        Returns:
+            dict: Details of the updated event.
+        """
+        client = await self.get_client()
+        event = await asyncio.to_thread(client.events().update, calendarId=calendar_id, eventId=event_id, body=updated_event)
+        return event.execute()
+
+    async def delete_event(self, calendar_id: str, event_id: str):
+        """
+        Delete an event from the specified calendar.
+
+        Args:
+            calendar_id (str): The ID of the calendar.
+            event_id (str): The ID of the event to delete.
+        """
+        client = await self.get_client()
+        await asyncio.to_thread(client.events().delete(calendarId=calendar_id, eventId=event_id).execute)
+        print(f"Event {event_id} deleted.")
+
+    async def get_event(self, calendar_id: str, event_id: str) -> Dict:
+        """
+        Retrieve details of a specific event.
+
+        Args:
+            calendar_id (str): The ID of the calendar.
+            event_id (str): The ID of the event.
+
+        Returns:
+            dict: Details of the event.
+        """
+        client = await self.get_client()
+        event = await asyncio.to_thread(client.events().get(calendarId=calendar_id, eventId=event_id).execute)
+        return event
+
+    async def setup_watch(self, calendar_id: str, webhook_url: str, channel_id: str) -> Dict:
+        """
+        Sets up a watch on the specified calendar to receive notifications when events are created, updated, or deleted.
+        
+        Args:
+            calendar_id (str): The ID of the calendar to monitor.
+            webhook_url (str): The URL of the webhook endpoint to receive notifications.
+            channel_id (str): Unique ID for this notification channel.
+        
+        Returns:
+            dict: The response from the Google Calendar API containing the channel information.
+        """
+        # Define request body for the watch request
+        request_body = {
+            "id": channel_id,  # Unique identifier for the channel
+            "type": "webhook",
+            "address": webhook_url,  # Webhook URL to receive notifications
+            "params": {
+                "ttl": "86400"  # Time-to-live in seconds, max 604800 (7 days)
+            }
+        }
+        if not self.service:
+            await self.get_client()
+        
+        try:
+            # Set up the watch on the specified calendar
+            response = self.service.events().watch(calendarId=calendar_id, body=request_body).execute()
+            print("Watch setup successful:", response)
+            return response
+        except Exception as e:
+            print(f"Error setting up watch: {e}")
+            raise
+
+    async def check_event_start_time(self, event_start_time: datetime):
+        """
+        Check if the current time has reached or passed the event's start time and trigger an action if so.
+        
+        Args:
+            event_start_time (datetime): The start time of the event to check.
+        
+        """
+        while True:
+            now = datetime.utcnow()
+            if now >= event_start_time:
+                # Trigger the action when event time is reached
+                print("Event time has been reached! Triggering action...")
+                # You can add your custom action here
+                break
+            await asyncio.sleep(60)  # Check every minute
+
+    async def create_subscription(self, webhook_url: str, client_state: str = "secret_string", expiration_hours: int = 1) -> dict:
+        """
+        Create a subscription to receive notifications when events are created, updated, or deleted in the calendar.
+
+        Args:
+            webhook_url (str): The webhook URL that will receive the notifications.
+            client_state (str): A client secret string for verifying notifications.
+            expiration_hours (int): Duration for which the subscription should be valid (maximum is 4230 minutes or 7 days).
+
+        Returns:
+            dict: The response from Microsoft Graph API with subscription details.
+        """
+        # Set up expiration for subscription (max 7 days)
+        expiration_date = datetime.utcnow() + timedelta(hours=expiration_hours)
+        expiration_datetime = expiration_date.isoformat() + "Z"
+
+        # Define the subscription request body
+        request_body = {
+            "changeType": "created,updated,deleted",
+            "notificationUrl": webhook_url,
+            "resource": "me/events",  # Subscribe to the user's calendar events
+            "expirationDateTime": expiration_datetime,
+            "clientState": client_state
+        }
+
+        # Acquire access token for authentication
+        access_token = self._access_token
+        headers = {
+            "Authorization": f"Bearer {access_token}",
+            "Content-Type": "application/json"
+        }
+
+        # Send the subscription request to Microsoft Graph API
+        url = "https://graph.microsoft.com/v1.0/subscriptions"
+        response = requests.post(url, headers=headers, json=request_body)
+
+        # Check for successful response
+        if response.status_code == 201:
+            subscription_info = response.json()
+            print("Subscription created successfully:", subscription_info)
+            return subscription_info
+        else:
+            error_message = response.json().get("error", {}).get("message", "Unknown error")
+            print(f"Failed to create subscription: {error_message}")
+            raise ComponentError(f"Failed to create subscription: {error_message}")

flowtask/interfaces/GoogleClient.py

@@ -0,0 +1,133 @@
+from pathlib import Path, PurePath
+from typing import Union, List, Dict, Callable
+from abc import ABC
+from oauth2client.service_account import ServiceAccountCredentials
+from googleapiclient.discovery import build
+from navconfig import BASE_DIR
+from ..exceptions import ComponentError, ConfigError
+from ..conf import GOOGLE_CREDENTIALS_FILE
+
+
+# Define the scope
+default_scopes = [
+    "https://spreadsheets.google.com/feeds",
+    "https://www.googleapis.com/auth/drive"
+]
+
+
+class GoogleClient(ABC):
+    """
+    Google Client Client.
+
+    Managing Authentication and resources from Google Apps.
+
+    """
+    def __init__(self, *args, credentials: Union[str, dict] = None, **kwargs):
+        self.credentials_file: PurePath = None
+        self.credentials_str: str = None
+        self.credentials_dict: dict = None
+        self.scopes: list = kwargs.pop('scopes', default_scopes)
+        if credentials is None:
+            if not GOOGLE_CREDENTIALS_FILE.exists():
+                raise ComponentError(
+                    "Google: No credentials provided."
+                )
+            self.credentials_file = GOOGLE_CREDENTIALS_FILE
+        if isinstance(credentials, str):
+            # end with JSON, then are a credentials file:
+            if credentials.endswith(".json"):
+                self.credentials_file = Path(credentials).resolve()
+                if not self.credentials_file.exists():
+                    # Check if File is on BASE PATH env.
+                    self.credentials_file = BASE_DIR.joinpath(credentials).resolve()
+                    if not self.credentials_file.exists():
+                        raise ConfigError(
+                            f"Google: Credentials file not found: {self.credentials_file}"
+                        )
+            else:
+                # Assume is a JSON string
+                self.credentials_str = credentials
+        elif isinstance(credentials, PurePath):
+            self.credentials_file = Path(credentials).resolve()
+            if not self.credentials_file.exists():
+                raise ConfigError(
+                    f"Google: No credentials file on {self.credentials_file}"
+                )
+        elif isinstance(credentials, dict):
+            self.credentials_dict = credentials
+
+        super().__init__(*args, **kwargs)
+
+    def connection(self):
+        if self.credentials_file:
+            self.credentials = ServiceAccountCredentials.from_json_keyfile_name(
+                self.credentials_file,
+                scopes=self.scopes,
+            )
+        elif self.credentials_dict:
+            self.credentials = ServiceAccountCredentials.from_json_keyfile_dict(
+                self.credentials_dict,
+                scopes=self.scopes
+            )
+        elif self.credentials_str:
+            self.credentials = ServiceAccountCredentials.from_json(
+                self.credentials_str,
+                scopes=self.scopes
+            )
+        else:
+            self.credentials = ServiceAccountCredentials.from_json_keyfile_name(
+                GOOGLE_CREDENTIALS_FILE,
+                scopes=self.scopes
+            )
+        return self
+
+    def get_service(self, service: str, version: str = 'v3'):
+        """
+        Get a cached Google API service instance or create one if not cached.
+
+        Args:
+            service (str): Name of the Google service (e.g., 'drive', 'sheets').
+            version (str): Version of the API (default: 'v3').
+
+        Returns:
+            googleapiclient.discovery.Resource: The requested Google API service client.
+        """
+        if not self.credentials:
+            self.connection()
+        if (srv := getattr(self, f"_{service}", None)):
+            return srv
+        srv = build(service, version, credentials=self.credentials)
+        setattr(self, f"_{service}", srv)
+        return srv
+
+    def get_drive_client(self):
+        """Shortcut for accessing the Google Drive client."""
+        return self.get_service("drive", "v3")
+
+    def get_sheets_client(self):
+        """Shortcut for accessing the Google Sheets client."""
+        return self.get_service("sheets", "v4")
+
+    def close(self):
+        """Clears cached credentials and services."""
+        self.credentials = None
+        self._services_cache.clear()
+
+    def get_search(self, query: str, version: str = 'v1', cse_id: str = None, **kwargs):
+        """
+        Get a cached Google API service instance or create one if not cached.
+
+        Args:
+
+            query (str): The search query.
+            version (str): Version of the API (default: 'v1').
+            cse_id (str): The Custom Search Engine ID.
+            **kwargs: Additional arguments for the API request.
+
+        Returns:
+            googleapiclient.discovery.Resource: The requested Google API service client.
+        """
+        if not self.credentials:
+            self.connection()
+        srv = build("customsearch", version, credentials=self.credentials)
+        return srv.cse().list(q=query, cx=cse_id, **kwargs)

flowtask/interfaces/GoogleDrive.py

@@ -0,0 +1,127 @@
+from abc import ABC
+from typing import Union
+from pathlib import Path, PurePath
+import asyncio
+import aiofiles
+import pandas as pd
+from googleapiclient.errors import HttpError
+from googleapiclient.http import MediaIoBaseDownload
+from .GoogleClient import GoogleClient
+from ..exceptions import ComponentError
+
+
+class GoogleDriveClient(GoogleClient, ABC):
+    """
+    Google Drive Client for downloading files from Google Drive.
+    """
+
+    async def download_file(
+        self,
+        source_filename: str,
+        destination_dir: Union[str, Path] = "."
+    ) -> PurePath:
+        """
+        Download a file from Google Drive by its name.
+
+        Args:
+            source_filename (str): The name of the file to download.
+            destination_dir (str or Path): Directory where the file will be saved (default is current directory).
+
+        Returns:
+            str: Path to the downloaded file.
+        """
+        try:
+            drive_service = await asyncio.to_thread(self.get_drive_client)
+
+            # Search for the file by name
+            results = await asyncio.to_thread(
+                drive_service.files().list,
+                q=f"name='{source_filename}'",
+                fields="files(id, name)",
+                pageSize=1
+            )
+            files = results.execute().get('files', [])
+            if not files:
+                raise ComponentError(f"File '{source_filename}' not found on Google Drive.")
+
+            file_id = files[0]['id']
+            request = drive_service.files().get_media(fileId=file_id)
+            file_path = Path(destination_dir) / source_filename
+
+            # Asynchronously open the file with aiofiles
+            async with aiofiles.open(file_path, 'wb') as f:
+                downloader = MediaIoBaseDownload(f, request)
+                done = False
+                while not done:
+                    # Run the next_chunk call in a separate thread
+                    status, done = await asyncio.to_thread(downloader.next_chunk)
+                    print(f"Download {int(status.progress() * 100)}%.")
+
+            return file_path
+
+        except HttpError as error:
+            raise ComponentError(f"Error downloading file from Google Drive: {error}")
+
+    async def download_folder(
+        self,
+        folder_name: str,
+        destination_dir: Union[str, Path] = "."
+    ) -> None:
+        """
+        Download all files within a specified Google Drive folder by name.
+
+        Args:
+            folder_name (str): The name of the folder to download.
+            destination_dir (str or Path): Directory where the files will be saved.
+
+        Returns:
+            None
+        """
+        try:
+            drive_service = await asyncio.to_thread(self.get_drive_client)
+
+            # Search for the folder by name to get the folder ID
+            folder_results = await asyncio.to_thread(
+                drive_service.files().list,
+                q=f"name='{folder_name}' and mimeType='application/vnd.google-apps.folder'",
+                fields="files(id, name)",
+                pageSize=1
+            )
+            folders = folder_results.execute().get('files', [])
+            if not folders:
+                raise ComponentError(f"Folder '{folder_name}' not found on Google Drive.")
+
+            folder_id = folders[0]['id']
+            destination_dir = Path(destination_dir) / folder_name
+            destination_dir.mkdir(parents=True, exist_ok=True)
+
+            # List all files in the folder
+            file_results = await asyncio.to_thread(
+                drive_service.files().list,
+                q=f"'{folder_id}' in parents and mimeType != 'application/vnd.google-apps.folder'",
+                fields="files(id, name)"
+            )
+            files = file_results.execute().get('files', [])
+
+            if not files:
+                print(f"No files found in folder '{folder_name}'.")
+                return
+
+            # Download each file in the folder
+            for file_info in files:
+                file_id = file_info['id']
+                file_name = file_info['name']
+                file_path = destination_dir / file_name
+
+                request = drive_service.files().get_media(fileId=file_id)
+                async with aiofiles.open(file_path, 'wb') as f:
+                    downloader = MediaIoBaseDownload(f, request)
+                    done = False
+                    while not done:
+                        status, done = await asyncio.to_thread(downloader.next_chunk)
+                        print(f"Downloading '{file_name}' - {int(status.progress() * 100)}% complete.")
+
+                print(f"Downloaded '{file_name}' to '{file_path}'.")
+
+        except HttpError as error:
+            raise ComponentError(f"Error downloading folder '{folder_name}' from Google Drive: {error}")

flowtask/interfaces/GoogleGCS.py

@@ -0,0 +1,89 @@
+from abc import ABC
+from pathlib import Path
+from typing import Union
+import asyncio
+from google.cloud import storage
+from google.auth.exceptions import GoogleAuthError
+from .GoogleClient import GoogleClient
+from ..exceptions import ComponentError
+
+
+class GoogleCloudStorageClient(GoogleClient, ABC):
+    """
+    Google Cloud Storage Client for interacting with Google Cloud Storage (GCS).
+    Provides methods for file and folder operations.
+    """
+
+    def __init__(self, *args, bucket_name: str, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.bucket_name = bucket_name
+        self._bucket = None
+
+    async def get_bucket(self):
+        """Get the GCS bucket, with caching."""
+        if not self._bucket:
+            try:
+                client = await asyncio.to_thread(storage.Client, credentials=self.credentials)
+                self._bucket = client.bucket(self.bucket_name)
+            except GoogleAuthError as e:
+                raise ComponentError(f"Google GCS authentication error: {e}")
+        return self._bucket
+
+    async def create_folder(self, folder_name: str):
+        """Create a folder in GCS by creating an empty blob with a trailing '/'."""
+        bucket = await self.get_bucket()
+        blob = bucket.blob(f"{folder_name}/")
+        await asyncio.to_thread(blob.upload_from_string, "")
+        print(f"Folder '{folder_name}' created in GCS.")
+
+    async def upload_file(self, source_path: Union[str, Path], destination_path: str):
+        """Upload a file to GCS."""
+        bucket = await self.get_bucket()
+        blob = bucket.blob(destination_path)
+        await asyncio.to_thread(blob.upload_from_filename, str(source_path))
+        print(f"File '{source_path}' uploaded to '{destination_path}' in GCS.")
+
+    async def download_file(self, source_path: str, destination_dir: Union[str, Path]):
+        """Download a file from GCS."""
+        bucket = await self.get_bucket()
+        blob = bucket.blob(source_path)
+        destination_file = Path(destination_dir) / Path(source_path).name
+        await asyncio.to_thread(blob.download_to_filename, str(destination_file))
+        print(f"File '{source_path}' downloaded to '{destination_file}'.")
+
+    async def delete_file(self, file_path: str):
+        """Delete a file from GCS."""
+        bucket = await self.get_bucket()
+        blob = bucket.blob(file_path)
+        await asyncio.to_thread(blob.delete)
+        print(f"File '{file_path}' deleted from GCS.")
+
+    async def upload_folder(self, source_folder: Union[str, Path], destination_folder: str):
+        """Upload all files from a local folder to a GCS folder."""
+        source_folder = Path(source_folder)
+        tasks = []
+        for file_path in source_folder.glob("**/*"):
+            if file_path.is_file():
+                relative_path = file_path.relative_to(source_folder)
+                destination_path = f"{destination_folder}/{relative_path}"
+                tasks.append(self.upload_file(file_path, destination_path))
+        await asyncio.gather(*tasks)
+        print(f"Folder '{source_folder}' uploaded to '{destination_folder}' in GCS.")
+
+    async def download_folder(self, source_folder: str, destination_folder: Union[str, Path]):
+        """Download all files from a GCS folder to a local folder."""
+        bucket = await self.get_bucket()
+        blobs = bucket.list_blobs(prefix=f"{source_folder}/")
+        destination_folder = Path(destination_folder) / source_folder
+        destination_folder.mkdir(parents=True, exist_ok=True)
+
+        tasks = []
+        async for blob in blobs:
+            if not blob.name.endswith("/"):  # Skip "directory" entries
+                relative_path = Path(blob.name).relative_to(source_folder)
+                local_file_path = destination_folder / relative_path
+                local_file_path.parent.mkdir(parents=True, exist_ok=True)
+                tasks.append(asyncio.to_thread(blob.download_to_filename, str(local_file_path)))
+        await asyncio.gather(*tasks)
+        print(f"Folder '{source_folder}' downloaded to '{destination_folder}'.")
+