flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/interfaces/Boto3Client.py

@@ -0,0 +1,326 @@
+from typing import Dict, List, Optional
+import re
+import boto3
+from botocore.exceptions import ClientError
+import aiofiles
+from navconfig.logging import logging
+from io import BytesIO
+from ..conf import aws_region
+try:
+    from settings.settings import AWS_CREDENTIALS
+except ImportError as e:
+    logging.exception(
+        f"AWS_CREDENTIALS not found. Please check your settings file: {e}"
+    )
+    from ..conf import AWS_CREDENTIALS
+from .client import ClientInterface
+from ..exceptions import FileNotFound, FileError, ComponentError
+
+
+logging.getLogger("boto3").setLevel(logging.CRITICAL)
+logging.getLogger("botocore").setLevel(logging.CRITICAL)
+logging.getLogger("s3transfer").setLevel(logging.CRITICAL)
+logging.getLogger("urllib3").setLevel(logging.CRITICAL)
+
+
+class Boto3Client(ClientInterface):
+    """
+    Boto3 AWS Client.
+
+        Overview
+
+        Abstract class for interaction with Boto3 (AWS).
+
+        .. table:: Properties
+        :widths: auto
+
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    | Name                   | Required | Summary                                                           |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    |_credentials            |   Yes    | The function is loaded and then we define the necessary code to   |
+    |                        |          | call the script                                                   |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    |  _init_                |   Yes    | Component for Data Integrator                                     |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    |  _host                 |   Yes    | The IPv4 or domain name of the server                             |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    |  get_client            |   Yes    | Gets the client access credentials, by which the user logs in to  |
+    |                        |          | perform an action                                                 |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    |  print                 |   Yes    | Print message to display                                          |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    | get_env_value          |   Yes    | Get env value  policies for setting virtual environment           |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+    | processing_credentials |   Yes    | client credentials configured for used of the app                 |
+    +------------------------+----------+-----------+-------------------------------------------------------+
+
+            Return the list of arbitrary days
+
+
+    """  # noqa
+
+    _credentials: Dict = {
+        "aws_key": str,
+        "aws_secret": str,
+        "client_id": str,
+        "client_secret": str,
+        "service": str,
+        "region_name": str,
+        "bucket": str,
+    }
+
+    def __init__(self, *args, **kwargs) -> None:
+        self.region_name: str = kwargs.pop('region_name', None)
+        self.service: str = kwargs.pop('service', 's3')
+        self.bucket: Optional[str] = kwargs.pop('bucket', None)
+        self.ContentType: str = kwargs.pop('ContentType', 'application/octet-stream')
+        if 'config_section' in kwargs:
+            self._config = kwargs.pop('config_section', 'default')
+        else:
+            self._config: str = kwargs.pop('config', 'default')
+        super().__init__(*args, **kwargs)
+
+    def define_host(self):
+        return True
+
+    async def open(
+        self,
+        host: str = None,
+        port: int = None,
+        credentials: dict = None,
+        **kwargs
+    ):
+        use_credentials = credentials.pop('use_credentials', True) if credentials else True
+        service = self.credentials.get('service', self.service)
+
+        if use_credentials is False:
+            self._logger.notice("Boto3: Using default credentials from environment")
+            self._connection = boto3.client(
+                service,
+                region_name=self.credentials.get("region_name", self.region_name)
+            )
+        else:
+            self._logger.notice("Boto3: Enter signed with explicit credentials")
+            cred = {
+                "aws_access_key_id": self.credentials["aws_key"],
+                "aws_secret_access_key": self.credentials["aws_secret"],
+                "region_name": self.credentials["region_name"],
+            }
+            self._connection = boto3.client(
+                service,
+                **cred
+            )
+        return self
+
+    def processing_credentials(self):
+        # getting credentials from self.credentials:
+        if self.credentials:
+            super().processing_credentials()
+        else:
+            # getting credentials from config
+            self.credentials = AWS_CREDENTIALS.get(self._config)
+        if not self.credentials:
+            raise ValueError(
+                f'Credentials not found for section {self._config}.'
+            )
+        ## getting Tenant and Site from credentials:
+        self.region_name = self.credentials.get('region_name', aws_region)
+        self.bucket = self.credentials.get('bucket_name', self.bucket)
+        self.service = self.credentials.get('service', 's3')
+
+    async def get_s3_object(self, bucket: str, filename: str):
+        """
+        Retrieve an object from an S3 bucket.
+
+        Parameters
+        ----------
+        bucket: str
+            The name of the S3 bucket.
+        filename: str
+            The name of the file (key) in the S3 bucket.
+
+        Returns
+        -------
+        dict
+            A dictionary containing the object data and metadata.
+
+        Raises
+        ------
+        FileNotFound
+            If the object is not found in the bucket.
+        ComponentError
+            If there is an issue with retrieving the object.
+        """
+        # Ensure connection is established
+        if not self._connection:
+            raise ComponentError(
+                "S3 client is not connected. Call `open` first."
+            )
+
+        # Get the object from S3
+        obj = self._connection.get_object(Bucket=bucket, Key=filename)
+        # Validate the response
+        status_code = int(obj["ResponseMetadata"]["HTTPStatusCode"])
+        if status_code != 200:
+            raise FileNotFound(
+                f"File '{filename}' not found in bucket '{bucket}'."
+            )
+        return obj
+
+    async def download_file(self, filename, obj):
+        result = None
+        ob_info = obj["ResponseMetadata"]["HTTPHeaders"]
+        rsp = obj["ResponseMetadata"]
+        status_code = int(rsp["HTTPStatusCode"])
+        if status_code == 200:
+            print('Content  ', ob_info["content-type"])
+            filepath = self.directory.joinpath(filename)
+            if ob_info["content-type"] == self.ContentType:
+                contenttype = ob_info["content-type"]
+
+                # Usar BytesIO solo cuando es un archivo individual
+                if hasattr(self, '_srcfiles') and not self._srcfiles:
+                    # Para múltiples archivos, usar streaming
+                    async with aiofiles.open(filepath, mode="wb") as fp:
+                        with obj["Body"] as stream:
+                            chunk_size = 8192  # 8KB chunks
+                            while True:
+                                chunk = stream.read(chunk_size)
+                                if not chunk:
+                                    break
+                                await fp.write(chunk)
+                    result = {
+                        "type": contenttype,
+                        "file": filepath
+                    }
+                else:
+                    # Para un archivo individual, mantener BytesIO
+                    data = None
+                    with obj["Body"] as stream:
+                        data = stream.read()
+                    output = BytesIO()
+                    output.write(data)
+                    output.seek(0)
+                    result = {"type": contenttype, "data": output, "file": filepath}
+                    # then save it into directory
+                    await self.save_attachment(filepath, data)
+            else:
+                return FileError(
+                    f'S3: Wrong File type: {ob_info["content-type"]!s}'
+                )
+        else:
+            return FileNotFound(
+                f"S3: File {filename} was not found: {rsp!s}"
+            )
+        return result
+
+    async def save_attachment(self, filepath, content):
+        try:
+            self._logger.info(f"S3: Saving attachment file: {filepath}")
+            if filepath.exists() is True:
+                if (
+                    "replace" in self.destination and self.destination["replace"] is True
+                ):
+                    # overwrite only if replace is True
+                    async with aiofiles.open(filepath, mode="wb") as fp:
+                        await fp.write(content)
+                else:
+                    self._logger.warning(
+                        f"S3: File {filepath!s} was not saved, already exists."
+                    )
+            else:
+                # saving file:
+                async with aiofiles.open(filepath, mode="wb") as fp:
+                    await fp.write(content)
+        except Exception as err:
+            raise FileError(f"File {filepath} was not saved: {err}") from err
+
+    async def close(self, **kwargs):
+        self._connection = None
+
+    async def _list_objects_with_pagination(self, kwargs: dict) -> List:
+        """Helper method to handle S3 pagination"""
+        all_objects = []
+        while True:
+            response = self._connection.list_objects_v2(**kwargs)
+            if response["KeyCount"] == 0:
+                if not all_objects:  # Solo lanzar error si no hay objetos
+                    raise FileNotFound(
+                        f"S3 Bucket Error: Content not found on {self.bucket}"
+                    )
+                break
+
+            all_objects.extend(response.get("Contents", []))
+
+            if not response.get("IsTruncated"):  # No hay más resultados
+                break
+
+            kwargs["ContinuationToken"] = response["NextContinuationToken"]
+
+        return all_objects
+
+    async def s3_list(self, suffix: str = "") -> List:
+        kwargs = {
+            "Bucket": self.bucket,
+            "Delimiter": "/",
+            "Prefix": self.source_dir,
+            "MaxKeys": 1000
+        }
+        prefix = self.source_dir
+        files = []
+        _patterns = []
+
+        if not self._srcfiles:
+            _patterns.append(re.compile(f"^{self.source_dir}.{suffix}+$"))
+            # List objects in the S3 bucket with the specified prefix
+            objects = await self._list_objects_with_pagination(kwargs)
+
+            for obj in objects:
+                key = obj["Key"]
+                if obj["Size"] == 0:
+                    # is a directory
+                    continue
+                if suffix is not None:
+                    if key.startswith(prefix) and re.match(
+                        prefix + suffix, key
+                    ):
+                        files.append(obj)
+                else:
+                    try:
+                        for pat in _patterns:
+                            mt = pat.match(key)
+                            if mt:
+                                files.append(obj)
+                    except Exception as e:
+                        self._logger.exception(e, stack_info=True)
+
+        if self._srcfiles:
+            for file in self._srcfiles:
+                _patterns.append(re.compile(f"^{self.source_dir}.{file}+$"))
+                objects = await self._list_objects_with_pagination(kwargs)
+
+                for obj in objects:
+                    key = obj["Key"]
+                    if obj["Size"] == 0:
+                        continue
+                    try:
+                        if hasattr(self, "source") and "filename" in self.source:
+                            if self.source["filename"] == key:
+                                files.append(obj)
+                    except (KeyError, AttributeError):
+                        pass
+                    if suffix is not None:
+                        if key.startswith(prefix) and re.match(
+                            prefix + suffix, key
+                        ):
+                            files.append(obj)
+                    else:
+                        try:
+                            for pat in _patterns:
+                                mt = pat.match(key)
+                                if mt:
+                                    files.append(obj)
+                        except Exception as e:
+                            self._logger.exception(e, stack_info=True)
+
+        return files

flowtask/interfaces/DropboxClient.py

@@ -0,0 +1,173 @@
+from abc import ABC
+from typing import Union
+from pathlib import Path
+import asyncio
+import aiofiles
+import dropbox
+from dropbox.exceptions import ApiError
+from dropbox.files import WriteMode
+from ..exceptions import ComponentError
+
+
+class DropboxClient(ABC):
+    """
+    Dropbox Client for downloading and uploading files and folders to/from Dropbox.
+    """
+    def __init__(
+        self,
+        *args,
+        access_key: Union[str, dict] = None,
+        **kwargs
+    ):
+        self.access_key: str = access_key
+        self.dbx = dropbox.Dropbox(self.access_key)
+        super().__init__(*args, **kwargs)
+
+    async def download_file(
+        self,
+        source_filename: str,
+        destination_dir: Union[str, Path] = "."
+    ) -> Path:
+        """
+        Download a file from Dropbox by its name.
+        """
+        try:
+            search_result = await asyncio.to_thread(
+                self.dbx.files_search_v2,
+                query=source_filename
+            )
+            if not search_result.matches:
+                raise ComponentError(
+                    f"File '{source_filename}' not found in Dropbox."
+                )
+
+            file_path = Path(destination_dir) / source_filename
+            file_metadata = search_result.matches[0].metadata
+            file_id = file_metadata.metadata.id
+
+            async with aiofiles.open(file_path, 'wb') as f:
+                _, res = await asyncio.to_thread(self.dbx.files_download, file_id)
+                await f.write(res.content)
+
+            return file_path
+
+        except ApiError as error:
+            raise ComponentError(
+                f"Error downloading file from Dropbox: {error}"
+            )
+
+    async def download_folder(
+        self,
+        folder_name: str,
+        destination_dir: Union[str, Path] = "."
+    ) -> None:
+        """
+        Download all files within a specified Dropbox folder by name.
+        """
+        try:
+            search_result = await asyncio.to_thread(
+                self.dbx.files_search_v2,
+                query=folder_name
+            )
+            if not search_result.matches:
+                raise ComponentError(
+                    f"Folder '{folder_name}' not found in Dropbox."
+                )
+
+            folder_metadata = search_result.matches[0].metadata
+            folder_id = folder_metadata.metadata.id
+            destination_dir = Path(destination_dir) / folder_name
+            destination_dir.mkdir(parents=True, exist_ok=True)
+
+            entries = await asyncio.to_thread(
+                self.dbx.files_list_folder,
+                folder_id
+            )
+
+            for entry in entries.entries:
+                if isinstance(entry, dropbox.files.FileMetadata):
+                    file_name = entry.name
+                    file_path = destination_dir / file_name
+
+                    async with aiofiles.open(file_path, 'wb') as f:
+                        _, res = await asyncio.to_thread(self.dbx.files_download, entry.id)
+                        await f.write(res.content)
+
+                    print(
+                        f"Downloaded '{file_name}' to '{file_path}'."
+                    )
+
+        except ApiError as error:
+            raise ComponentError(
+                f"Error downloading folder '{folder_name}' from Dropbox: {error}"
+            )
+
+    async def upload_file(
+        self,
+        source_filepath: Union[str, Path],
+        destination_path: Union[str, Path] = "/"
+    ) -> None:
+        """
+        Upload a file to Dropbox.
+        """
+        source_filepath = Path(source_filepath)
+        if not source_filepath.exists() or not source_filepath.is_file():
+            raise ComponentError(
+                f"Source file '{source_filepath}' does not exist."
+            )
+
+        try:
+            async with aiofiles.open(source_filepath, 'rb') as f:
+                file_data = await f.read()
+                destination_path = str(Path(destination_path) / source_filepath.name)
+
+                await asyncio.to_thread(
+                    self.dbx.files_upload,
+                    file_data,
+                    destination_path,
+                    mode=WriteMode.overwrite
+                )
+                print(
+                    f"Uploaded file '{source_filepath.name}' to '{destination_path}'."
+                )
+
+        except ApiError as error:
+            raise ComponentError(
+                f"Error uploading file '{source_filepath}': {error}"
+            )
+
+    async def upload_folder(
+        self,
+        source_dir: Union[str, Path],
+        destination_path: Union[str, Path] = "/"
+    ) -> None:
+        """
+        Upload all files within a specified local folder to Dropbox.
+        """
+        source_dir = Path(source_dir)
+        if not source_dir.exists() or not source_dir.is_dir():
+            raise ComponentError(
+                f"Source directory '{source_dir}' does not exist."
+            )
+
+        for file_path in source_dir.rglob('*'):
+            if file_path.is_file():
+                # Preserve directory structure within Dropbox
+                relative_path = file_path.relative_to(source_dir)
+                dropbox_path = str(Path(destination_path) / relative_path)
+
+                try:
+                    async with aiofiles.open(file_path, 'rb') as f:
+                        file_data = await f.read()
+                        await asyncio.to_thread(
+                            self.dbx.files_upload,
+                            file_data,
+                            dropbox_path,
+                            mode=WriteMode.overwrite
+                        )
+                        print(f"Uploaded '{file_path}' to '{dropbox_path}'.")
+
+                except ApiError as error:
+                    raise ComponentError(
+                        f"Error uploading file '{file_path}': {error}"
+                    )

flowtask/interfaces/ExcelHandler.py

@@ -0,0 +1,94 @@
+from pathlib import Path
+from typing import Callable
+import pandas as pd
+from openpyxl import load_workbook, Workbook
+from openpyxl.utils.dataframe import dataframe_to_rows
+from openpyxl.workbook import Workbook as OpenpyxlWorkbook
+
+
+class ExcelHandler:
+    def pandas_to_excel(self, df: pd.DataFrame, file_path: Path, **kwargs):
+        """
+        Save a pandas DataFrame to an Excel file.
+
+        :param df: DataFrame to save.
+        :param file_path: Path to save the Excel file.
+        :param kwargs: Additional keyword arguments for pandas' to_excel method.
+        """
+        df.to_excel(file_path, index=False, **kwargs)
+
+    def edit_excel(self, file_path: Path, edit_func: Callable[[OpenpyxlWorkbook], None]):
+        """
+        Edit an existing Excel file using a provided function.
+
+        :param file_path: Path to the Excel file.
+        :param edit_func: Function that takes a Workbook object and edits it.
+        """
+        if not file_path.exists():
+            raise FileNotFoundError(f"The file {file_path} does not exist.")
+        wb = load_workbook(file_path)
+        edit_func(wb)
+        wb.save(file_path)
+
+    def append_data_to_excel(
+        self,
+        df: pd.DataFrame,
+        file_path: Path,
+        sheet_name: str,
+        start_row: int = None,
+        start_col: int = 1,
+        include_header: bool = False
+    ):
+        """
+        Append data from a DataFrame to an existing Excel file.
+
+        :param df: DataFrame containing data to append.
+        :param file_path: Path to the Excel file.
+        :param sheet_name: Name of the sheet to append data to.
+        :param start_row: Starting row to append data (default is after last row).
+        :param start_col: Starting column to append data.
+        :param include_header: Whether to include the DataFrame's header.
+        """
+        if not file_path.exists():
+            raise FileNotFoundError(f"The file {file_path} does not exist.")
+        wb = load_workbook(file_path)
+        if sheet_name not in wb.sheetnames:
+            ws = wb.create_sheet(title=sheet_name)
+            start_row = 1
+        else:
+            ws = wb[sheet_name]
+            if start_row is None:
+                start_row = ws.max_row + 1
+
+        for r_idx, row in enumerate(
+            dataframe_to_rows(df, index=False, header=include_header), start=start_row
+        ):
+            for c_idx, value in enumerate(row, start=start_col):
+                ws.cell(row=r_idx, column=c_idx, value=value)
+
+        wb.save(file_path)
+
+    def create_new_excel_file(self, file_path: Path):
+        """
+        Create a new Excel file.
+
+        :param file_path: Path where the new Excel file will be saved.
+        """
+        if file_path.exists():
+            raise FileExistsError(f"The file {file_path} already exists.")
+        wb = Workbook()
+        wb.save(file_path)
+
+    def create_excel_from_template(self, template_path: Path, new_file_path: Path):
+        """
+        Create a new Excel file from a template.
+
+        :param template_path: Path to the Excel template.
+        :param new_file_path: Path where the new Excel file will be saved.
+        """
+        if not template_path.exists():
+            raise FileNotFoundError(f"The template {template_path} does not exist.")
+        if new_file_path.exists():
+            raise FileExistsError(f"The file {new_file_path} already exists.")
+        wb = load_workbook(template_path)
+        wb.save(new_file_path)