flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/DownloadFromS3.py

@@ -0,0 +1,198 @@
+import os
+import asyncio
+import re
+from typing import List
+from collections.abc import Callable
+import aiofiles
+from botocore.exceptions import ClientError
+from ..exceptions import FileError, ComponentError, FileNotFound
+from .DownloadFrom import DownloadFromBase
+from ..interfaces.Boto3Client import Boto3Client
+
+
+class DownloadFromS3(Boto3Client, DownloadFromBase):
+    """
+    DownloadFromS3.
+
+    **Overview**
+
+        Download a file from an Amazon S3 bucket using the functionality from DownloadFrom.
+
+    **Properties**
+
+    .. table:: Properties
+        :widths: auto
+
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | Name               | Required | Summary                                                                     |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | credentials        |   Yes    | Credentials to establish connection with S3 service (username and password) |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | bucket             |   Yes    | The name of the S3 bucket to download files from.                           |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | source_dir         |   No     | The directory path within the S3 bucket to download files from.             |
+        |                    |          | Defaults to the root directory (`/`).                                       |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | source             |   No     | A dictionary specifying the filename to download.                           |
+        |                    |          | If provided, takes precedence over `source_dir` and `_srcfiles`.            |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | _srcfiles          |   No     | A list of filenames to download from the S3 bucket.                         |
+        |                    |          | Used in conjunction with `source_dir`.                                      |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | rename             |   No     | A new filename to use for the downloaded file.                              |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | directory          |   Yes    | The local directory path to save the downloaded files.                      |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+        | create_destination |   No     | A boolean flag indicating whether to create the destination directory       |
+        |                    |          | if it doesn't exist. Defaults to `True`.                                    |
+        +--------------------+----------+-----------+-----------------------------------------------------------------+
+
+        save the file on the new destination.
+
+    **Methods**
+
+    * start()
+    * close()
+    * run()
+    * s3_list(s3_client, suffix="")
+    * save_attachment(self, filepath, content)
+    * download_file(self, filename, obj)
+    
+
+        Example:
+
+        ```yaml
+        DownloadFromS3:
+          credentials:
+            use_credentials: false
+            region_name: us-east-2
+          bucket: placer-navigator-data
+          source_dir: placer-analytics/bulk-export/monthly-weekly/2025-03-06/metadata/
+          destination:
+            directory: /nfs/symbits/placerai/2025-03-06/metadata/
+          create_destination: true
+        ```
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.url: str = None
+        self.folder = None
+        self.rename: str = None
+        self.context = None
+        super().__init__(
+            loop=loop, job=job, stat=stat, **kwargs
+        )
+
+    async def start(self, **kwargs):
+        await super(DownloadFromS3, self).start(**kwargs)
+        if isinstance(self.directory, str) and "{" in self.directory:
+            self.directory = self.mask_replacement(self.directory)
+            print(f"Directory after mask replacement: {self.directory}")
+        if hasattr(self, 'source_dir') and self.source_dir and isinstance(self.source_dir, str) and "{" in self.source_dir:
+            self.source_dir = self.mask_replacement(self.source_dir)
+            print(f"Source directory after mask replacement: {self.source_dir}")
+        if self.source_dir and not self.source_dir.endswith("/"):
+            self.source_dir = self.source_dir + "/"
+        return True
+
+    async def run(self):
+        try:
+            if not self.directory.exists():
+                if self.create_destination is True:
+                    try:
+                        self.directory.mkdir(parents=True, exist_ok=True)
+                    except Exception as e:
+                        raise ComponentError(
+                            f"S3: Error creating destination directory: {e}"
+                        ) from e
+                else:
+                    raise ComponentError(
+                        f"S3: destination directory doesn't exists: {self.directory}"
+                    )
+        except Exception as err:
+            self._logger.error(f"S3: Error creating destination directory: {err}")
+            raise ComponentError(
+                f"S3: Error creating destination directory: {err}"
+            ) from err
+        errors = {}
+        files = {}
+        if not self._connection:
+            await self.open(credentials=self.credentials)
+        if hasattr(self, "file"):
+            # find src files using get_list:
+            s3_files = await self.s3_list(self._connection)
+            for obj in s3_files:
+                try:
+                    file = obj["Key"]
+                    try:
+                        obj = await self._connection.get_object(
+                            Bucket=self.bucket, Key=file
+                        )
+                    except Exception as e:
+                        raise ComponentError(
+                            f"S3: Error getting object from Bucket: {e}"
+                        ) from e
+                    result = await self.download_file(os.path.basename(file), obj)
+                    if isinstance(result, BaseException):
+                        errors[file] = result
+                    else:
+                        files[file] = result
+                except Exception as e:
+                    raise ComponentError(f"{e!s}") from e
+        else:
+            if not self._srcfiles:
+                # Download all files from source_dir
+                s3_files = await self.s3_list()
+                files = {}
+                for file in s3_files:
+                    try:
+                        # Extraer la Key del objeto S3
+                        file_key = file["Key"] if isinstance(file, dict) else file
+                        obj = await self.get_s3_object(
+                            bucket=self.bucket, filename=file_key
+                        )
+                    except FileNotFound:
+                        raise
+                    except Exception as e:
+                        raise ComponentError(
+                            f"S3: Error getting object from Bucket: {e}"
+                        ) from e
+                    result = await self.download_file(os.path.basename(file_key), obj)
+                    if isinstance(result, BaseException):
+                        errors[file_key] = result
+                    else:
+                        files[file_key] = result
+            else:
+                for file in self._srcfiles:
+                    if self.source_dir:
+                        filename = f"{self.source_dir}{file}"
+                    else:
+                        filename = file
+                    print(f"Downloading file: {filename}")
+                    try:
+                        self._logger.debug(f"S3: Downloading File {filename}")
+                        obj = await self.get_s3_object(
+                            bucket=self.bucket, filename=filename
+                        )
+                    except FileNotFound:
+                        raise
+                    except Exception as e:
+                        raise ComponentError(
+                            f"S3: Error getting object from Bucket: {e}"
+                        ) from e
+                    result = await self.download_file(file, obj)
+                    if isinstance(result, BaseException):
+                        errors[file] = result
+                    else:
+                        files[file] = result
+        # at end, create the result:
+        self._result = {"files": files, "errors": errors}
+        self.add_metric("S3_FILES", files)
+        return self._result

flowtask/components/DownloadFromSFTP.py

@@ -0,0 +1,265 @@
+import asyncio
+from collections.abc import Callable
+from pathlib import Path, PurePath
+import asyncssh
+from navconfig.logging import logging
+from ..exceptions import FileError, ComponentError, FileNotFound
+from .DownloadFrom import DownloadFromBase
+from ..interfaces.SSHClient import SSHClient
+
+
+class DownloadFromSFTP(SSHClient, DownloadFromBase):
+    """
+    DownloadFromSFTP.
+
+    **Overview**
+
+        Download a file or directory from an SFTP server using the functionality from DownloadFrom.
+
+    **Properties** (inherited from DownloadFromBase and SSHClient)
+
+    .. table:: Properties
+        :widths: auto
+
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | Name               | Required | Summary                                                                                |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | credentials        |   Yes    | Credentials to establish connection with SFTP server (username and password)           |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | host               |   Yes    | The hostname or IP address of the SFTP server.                                         |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | port               |   No     | The port number of the SFTP server (default: 22).                                      |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | tunnel             |   No     | Dictionary defining a tunnel to use for the connection.                                |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | block_size         |   No     | Block size for file transfer (default: 4096 bytes).                                    |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | max_requests       |   No     | Maximum number of concurrent file transfers (default: 5).                              |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | create_destination |   No     | Boolean flag indicating whether to create the destination directory                    |
+        |                    |          | if it doesn't exist (default: True).                                                   |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | source             |   Yes    | A dictionary specifying the source file or directory on the SFTP server.               |
+        |                    |          | Can include:                                                                           |
+        |                    |          |  whole_dir (Optional, bool): Whether to download the entire directory (default: False).|
+        |                    |          |  recursive (Optional, bool): Whether to download subdirectories recursively when       |
+        |                    |          |  `whole_dir` is True (default: False).                                                 |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | filename           |   No     | The filename to download from the SFTP server (if not using `source`).                 |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | mdate              |   No     | Modification date of the file to download (for filtering based on modification time).  |
+        +--------------------+----------+-----------------------+----------------------------------------------------------------+
+        | rename             |   No     | A new filename to use for the downloaded file.                                         |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | masks              |   No     | A dictionary mapping mask strings to replacement strings used for renaming files and   |
+        |                    |          | modification dates.                                                                    |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | overwrite          |   No     | Whether to overwrite existing files in the destination directory (default: False).     |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+        | remove             |   No     | Whether to remove the downloaded files from the SFTP server after                      |
+        |                    |          | successful download (default: False).                                                  |
+        +--------------------+----------+-----------+----------------------------------------------------------------------------+
+
+    Save the downloaded files on the new destination.
+
+    
+
+        Example:
+
+        ```yaml
+        DownloadFromSFTP:
+          file:
+            pattern: Performance_Tracker/*
+            mdate: '{today}'
+          host: altice_ltm_sftp_host
+          port: altice_ltm_sftp_port
+          credentials:
+            username: altice_ltm_sftp_username
+            password: altice_ltm_sftp_password
+            known_hosts: null
+          directory: /home/ubuntu/altice/files/
+          overwrite: true
+          masks:
+            '{today}':
+            - yesterday
+            - mask: '%Y-%m-%d'
+        ```
+
+    """  # noqa
+
+    to_remove = (
+        "file",
+        "filename",
+        "directory",
+        "source",
+        "destination",
+        "create_destination",
+        "overwrite",
+        "rename",
+    )
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.mdate = None
+        self.local_name = None
+        self.rename: str = None
+        self.filename: str = ""
+        self.whole_dir: bool = False
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        await super(DownloadFromSFTP, self).start(**kwargs)
+        if hasattr(self, "source"):
+            self.whole_dir = (
+                self.source["whole_dir"] if "whole_dir" in self.source else False
+            )
+        if hasattr(self, "file"):
+            if "mdate" in self.file:
+                self.mdate = self.file["mdate"]
+        # Mask in rename file
+        if hasattr(self, "masks"):
+            try:
+                for mask, replace in self._mask.items():
+                    if self.rename is not None:
+                        self.rename = self.rename.replace(mask, str(replace))
+                    if self.mdate is not None:
+                        self.mdate = self.mdate.replace(mask, replace)
+            except Exception as e:
+                raise ComponentError(f"{e!s}") from e
+            self.rename = Path("{}/{}".format(self.directory, self.rename))
+        if self.rename is not None:
+            self.local_name = self.rename
+        else:
+            self.local_name = Path(
+                "{}/{}".format(self.directory, PurePath(self.filename).name)
+            )
+        return True
+
+    def download_progress(self, srcpath, dstpath, bytes_copied, total_bytes):
+        self._filenames[dstpath] = srcpath
+        self._pb.reset(total=total_bytes)
+        self._pb.update(bytes_copied)
+        self._pb.refresh()
+
+    async def download(self):
+        try:
+            await self.open(
+                host=self.host,
+                port=self.port,
+                tunnel=self.tunnel,
+                credentials=self.credentials,
+            )
+        except asyncio.CancelledError:
+            self._logger.info(f"{self.host} CANCELED~")
+            # break
+        except (asyncio.TimeoutError, ComponentError) as err:
+            raise ComponentError(f"Error: {err}") from err
+        except Exception as err:
+            raise ComponentError(f"Error: {err}") from err
+        async with self._connection.start_sftp_client() as sftp:
+            # check all versions of functionalities
+            args = {
+                "block_size": self.block_size,
+                "max_requests": self.max_requests,
+                "progress_handler": self.download_progress,
+                "error_handler": self.err_handler,
+            }
+            if self.whole_dir is True:
+                if hasattr(self, "source"):  # using the source/destination filosophy
+                    args["remotepaths"] = self.source_dir
+                    args["localpath"] = self.directory
+                    args["recurse"] = True if "recursive" in self.source else False
+                else:
+                    raise ComponentError(
+                        "*Whole Dir* and self.source are mutually inclusive, \
+                            missing self.source"
+                    )
+                # getting the stats of the remote path:
+                stats = await sftp.listdir(path=self.source_dir)
+                self.start_progress(total=len(stats))
+            else:
+                filelist = []
+                if self._srcfiles:
+                    logging.debug(f"FILES::: {self._srcfiles}")
+                    for file in self._srcfiles:
+                        if self.mdate is not None:
+                            flist = await self.get_mtime_files(sftp, file, self.mdate)
+                            filelist += flist
+                            if not flist:
+                                raise FileNotFound(
+                                    f"File Not Found for modification date {self.mdate}"
+                                )
+                        else:
+                            filelist.append(file)
+                else:
+                    filelist = self.filename
+                args["remotepaths"] = filelist
+                args["localpath"] = self.directory
+                args["preserve"] = True
+                self.start_progress(total=len(filelist))
+            try:
+                await sftp.mget(**args)
+                self.close_progress()
+                # get all the files downloaded
+                result = list(self._filenames.keys())
+                self._result = [v.decode("utf8") for v in result]
+                # Rename file if only one
+                if len(self._result) == 1 and self.rename is not None:
+                    f = Path(self._result[0])
+                    f.rename(self.local_name)
+                    self._result[0] = self.local_name
+                if hasattr(self, "remove") and self.remove is True:
+                    try:
+                        for _, filename in self._filenames.items():
+                            logging.debug(f"Removing remote file: {filename}")
+                            await sftp.remove(filename)
+                    except asyncssh.sftp.SFTPError as err:
+                        logging.error(f"SFTP error: {err}")
+                return True
+            except OSError as err:
+                self._logger.error(err)
+                raise ComponentError(f"SFTP Local error: {err}") from err
+            except asyncssh.sftp.SFTPNoSuchFile as err:
+                raise FileNotFound(f"File Not Found: {err}") from err
+            except asyncssh.sftp.SFTPError as err:
+                self._logger.error(err)
+                raise FileError(f"SFTP server error: {err}") from err
+            except asyncssh.Error as err:
+                self._logger.error(err)
+                raise ComponentError(f"SFTP connection failed: {err}") from err
+            except FileNotFound:
+                raise
+            except Exception as err:
+                self._logger.exception(err, stack_info=True)
+                raise ComponentError(f"SFTP Error: {err}") from err
+
+    async def run(self):
+        """Running Download file."""
+        self._result = None
+        status = False
+        if self.overwrite is False and self.local_name.is_file():
+            # file already exists and can be skipped
+            self._filenames = [str(self.local_name)]
+            self._result = [str(self.local_name)]
+            status = True
+        else:
+            try:
+                status = await self.download()
+            except FileNotFound:
+                raise
+            except FileError:
+                raise
+            except ComponentError:
+                raise
+            except Exception as exc:
+                raise ComponentError(f"{exc}") from exc
+        if status is False:
+            return False
+        else:
+            self.add_metric("SFTP_FILES", self._result)
+            return self._result

flowtask/components/DownloadFromSharepoint.py

@@ -0,0 +1,110 @@
+import asyncio
+from collections.abc import Callable
+from navconfig.logging import logging
+from ..exceptions import FileNotFound, FileError
+from .DownloadFrom import DownloadFromBase
+from ..interfaces.Sharepoint import SharepointClient
+
+
+class DownloadFromSharepoint(SharepointClient, DownloadFromBase):
+    """
+    DownloadFromSharepoint.
+
+    **Overview**
+
+        This Sharepoint component downloads a file or uploads it to the Microsoft Sharepoint service
+
+    **Properties** (inherited from DownloadFromBase and Sharepoint)
+
+    .. table:: Properties
+        :widths: auto
+
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | Name               | Required | Summary                                                                          |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | credentials        |   Yes    | Credentials to establish connection with SharePoint site (username and password) |
+        |                    |          | get credentials from environment if null.                                        |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | site               |   Yes    | The URL of the SharePoint site.                                                  |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | verify             |   No     | Whether to verify the SSL certificate of the SharePoint site (default: True).    |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | timeout            |   No     | The timeout value for SharePoint operations (default: 30 seconds).               |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | create_destination |   No     | Boolean flag indicating whether to create the destination directory if it        |
+        |                    |          | doesn't exist (default: True).                                                   |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | tenant             |   Yes    | Is the set of site collections of SharePoint                                     |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+        | file_id            |   Yes    | Identificador del archivo en sharepoint                                          |
+        +--------------------+----------+-----------+----------------------------------------------------------------------+
+
+    Save the downloaded files on the new destination.
+    
+
+        Example:
+
+        ```yaml
+        DownloadFromSharepoint:
+          credentials:
+            username: SHAREPOINT_TROCADV_USERNAME
+            password: SHAREPOINT_TROCADV_PASSWORD
+            tenant: symbits
+            site: FlexRoc
+          file:
+            filename: Bose Schedule.xlsx
+            directory: Shared Documents/General/Monthly Schedule & Reporting
+          destination:
+            directory: /home/ubuntu/symbits/bose/stores/
+            filename: Bose Schedule - {today}.xlsx
+          masks:
+            '{today}':
+            - today
+            - mask: '%Y-%m-%d'
+        ```
+
+    """  # noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.url: str = None
+        self.folder = None
+        self.rename: str = None
+        self.context = None
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        # Call the start method from the base classes
+        await super(DownloadFromSharepoint, self).start(**kwargs)
+        self._started = True
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        async with self.connection():
+            if not self.context:
+                self.context = self.get_context(self.url)
+            try:
+                if hasattr(self, 'file') and "pattern" in self.file:  # search-like context
+                    filenames = await self.file_search()
+                else:
+                    filenames = await self.file_download()
+                self._result = filenames
+                self.add_metric(
+                    "SHAREPOINT_FILES",
+                    self._result
+                )
+                return self._result
+            except (FileError, FileNotFound):
+                raise
+            except Exception as err:
+                logging.error(f"Error downloading File: {err}")
+                raise FileError(
+                    f"Error downloading File: {err}"
+                ) from err