flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/DownloadFromFTP.py

@@ -0,0 +1,181 @@
+"""
+  DownloadFromSmartSheet
+  Download an Excel file from SmartSheet.
+
+
+        Example:
+
+        ```yaml
+        DownloadFromFTP:
+          host: trendmicro_host
+          port: trendmicro_port
+          use_tls: false
+          file:
+            pattern: '{monday}.*-Limited852StoreReport.{saturday}.csv'
+            values:
+              monday:
+              - date_dow
+              - day_of_week: monday
+                mask: '%Y%m%d'
+              saturday:
+              - date_diff_dow
+              - diff: 7
+                day_of_week: saturday
+                mask: '%Y-%m-%d'
+            directory: /Limited852Report
+          credentials:
+            user: trendmicro_user
+            password: trendmicro_password
+          download:
+            directory: /home/ubuntu/symbits/trendmicro/files/market_share/
+        ```
+
+    """
+import re
+import asyncio
+from pathlib import Path
+from ..exceptions import ComponentError, FileError
+from .DownloadFrom import DownloadFromBase
+from ..interfaces.FTPClient import FTPClient
+
+
+class DownloadFromFTP(FTPClient, DownloadFromBase):
+    """
+    DownloadFromFTP
+
+    Overview
+
+        Downloads files from a remote FTP server using the functionality from DownloadFrom.
+
+    Properties
+
+    .. table:: Properties
+        :widths: auto
+
+    +--------------+----------+-----------+--------------------------------------------------------+
+    | Name         | Required | Summary                                                            |
+    +--------------+----------+-----------+--------------------------------------------------------+
+    | credentials  |   Yes    | Dictionary containing username and password for FTP authentication.|
+    |              |          | (default: {"user": str, "password": str})                          |
+    +--------------+----------+-----------+--------------------------------------------------------+
+    | whole_dir    |   Yes    | Boolean flag indicating whether to download the entire directory   |
+    |              |          | or specific files. (default: False)                                |
+    +--------------+----------+-----------+--------------------------------------------------------+
+    | use_tls      |   Yes    | Boolean for indicate if we need to use the TLS protocol            |
+    +--------------+----------+-----------+--------------------------------------------------------+
+
+    Save the downloaded files on the new destination.
+    """
+    _credentials: dict = {"user": str, "password": str}
+
+    async def start(self, **kwargs):
+        await super().start(**kwargs)
+        self._path: str = None
+        self.whole_dir: bool = False
+        if hasattr(self, "source"):
+            if "algorithms" in self.source:
+                self.algorithms.append(self.source["algorithms"])
+            # change filosophy of source/destination
+            self.whole_dir = (
+                self.source["whole_dir"] if "whole_dir" in self.source else False
+            )
+        await super(DownloadFromFTP, self).start(**kwargs)
+        if hasattr(self, "download"):
+            self.directory = Path(self.download["directory"])
+        return True
+
+    async def download_files(self):
+        try:
+            self._connection = await self.init_connection(
+                self.host, self.port, credentials=self.credentials, ssl=self.ssl
+            )
+        except asyncio.CancelledError:
+            self._logger.info(f"{self.host} CANCELED~")
+            # break
+        except ComponentError:
+            raise
+        except (TimeoutError, asyncio.TimeoutError) as ex:
+            raise ComponentError(f"Timeout: {ex}") from ex
+        except Exception as err:
+            raise ComponentError(f"DownloadFromFTP Error: {err}") from err
+        if self._connection:
+            if not await self.directory_exists(self.source_dir):
+                raise ComponentError(f"FTP Directory doesn't exists: {self.source_dir}")
+            try:
+                await self.change_directory(self.source_dir)
+            except Exception as err:
+                raise ComponentError(
+                    f"FTP Unable to connect to: {self.source_dir}, error: {err}"
+                ) from err
+            # getting stats for directory
+            filelist = []
+            files = []
+            stats = await self._connection.list(
+                self.source_dir, recursive=False, raw_command="MLSD"
+            )
+            pbar = self.start_pbar(total=len(stats))
+            if self.whole_dir is True:
+                await self.download_file(
+                    file=self.source_dir, destination=self.directory
+                )
+                for path, info in stats:
+                    if info["type"] == "file":
+                        file = {"filename": path, **info}
+                        # all files downloaded:
+                        pbar.update(1)
+                        files.append(file)
+                        filelist.append(path)
+                self.add_metric("FTP_FILES", files)
+                pbar.close()
+            else:
+                if self.source_file is not None:
+                    self._srcfiles = [self.source_file]
+                for file in self._srcfiles:
+                    if not self.source_dir.endswith("/"):
+                        self.source_dir = "{}/".format(self.source_dir)
+                    pattern = re.compile(f"^{self.source_dir}{str(file)}+$")
+                    self._logger.debug(f"Calculated Pattern for lookup {pattern}")
+                    for path, info in stats:
+                        if info["type"] == "file":
+                            # first case, match the extension:
+                            if hasattr(self, "source"):
+                                if "suffix" in self.source:
+                                    # download all files match with suffix:
+                                    if path.suffix == self.source["suffix"]:
+                                        file = {"filename": path, **info}
+                                        await self.download_file(
+                                            file=path, destination=self.directory
+                                        )
+                                        files.append(file)
+                                        filelist.append(path)
+                            elif pattern.match(str(path)):
+                                self._logger.debug(f"Looking for matching file {path}")
+                                file = {"filename": path, **info}
+                                files.append(file)
+                                filelist.append(path)
+                                # using regex to check file
+                                await self.download_file(
+                                    file=path, destination=self.directory
+                                )
+                            pbar.update(1)
+                            pbar.set_description(f"Processing {path}")
+                self.add_metric("FTP_FILES", files)
+                pbar.close()
+            self._result = ["{}/{}".format(self.directory, v.name) for v in filelist]
+            print(self._result)
+            return True
+        else:
+            return False
+
+    async def run(self):
+        self._result = None
+        if self._debug:
+            self._logger.info(f"Downloading FTP files: {self._filenames}")
+        try:
+            status = await self.download_files()
+        except (ComponentError, Exception) as err:
+            self._logger.error(err)
+            raise
+        if not status:
+            raise FileError(f"File(s) Not Found: {self.source_file}")
+        return self._result

flowtask/components/DownloadFromIMAP.py

@@ -0,0 +1,315 @@
+import asyncio
+from collections.abc import Callable
+import re
+import fnmatch
+from pathlib import Path, PurePath
+import imaplib
+import aiofiles
+from ..conf import IMAP_RETRY_SELECT, FILES_PATH
+from ..utils.mail import MailMessage
+from ..exceptions import ComponentError, FileNotFound
+from .DownloadFrom import DownloadFromBase
+from ..interfaces.IMAPClient import IMAPClient
+
+
+class DownloadFromIMAP(IMAPClient, DownloadFromBase):
+    """
+    DownloadFromIMAP.
+
+        Overview
+
+            Download emails from an IMAP mailbox using the functionality from DownloadFrom.
+
+        .. table:: Properties
+            :widths: auto
+
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | Name              | Required | Summary                                                                                                 |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | credentials       |   Yes    | Credentials to access the IMAP mailbox.                                                                 |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | mailbox           |   Yes    | The IMAP mailbox name (default: "INBOX").                                                               |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | search_terms      |   Yes    | Dictionary containing search criteria in IMAP format.                                                   |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | attachments       |   Yes    | Dictionary specifying download configuration for attachments:                                           |
+            |                   |          |   - directory (str): Path to save downloaded attachments.                                               |
+            |                   |          |   - filename (str, optional): Filename pattern for selection (fnmatch).                                 |
+            |                   |          |   - pattern (str, optional): Regular expression pattern for selection.                                  |
+            |                   |          |   - expected_mime (str, optional): Expected MIME type filter.                                           |
+            |                   |          |   - rename (str, optional): Template string for renaming attachments (uses "{filename}").               |
+            |                   |          |   - download_existing (bool, optional): Skip existing files (default: True).                            |
+            |                   |          |   - create_destination (bool, optional): Create download directory if it doesn't exist (default: True). |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | download_existing |    no    | Flag indicating whether to skip downloading existing files.                                             |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | results           |   Yes    | Dictionary containing download results:                                                                 |
+            |                   |          |   - attachments: List of downloaded attachment file paths.                                              |
+            |                   |          |   - messages: List of `MailMessage` objects representing downloaded emails.                             |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+            | use_ssl           |   Yes    | Boolean for indicate if we need to use the TLS protocol                                                 |
+            +-------------------+----------+-----------+---------------------------------------------------------------------------------------------+
+
+            Save the downloaded files on the new destination.
+    
+
+        Example:
+
+        ```yaml
+        DownloadFromIMAP:
+          credentials:
+            host: email_host
+            port: email_port
+            user: email_host_user
+            password: email_host_password
+            use_ssl: true
+          search_terms:
+            'ON': '{search_today}'
+            SUBJECT: Custom Punch with Pay Codes - Excel
+            FROM: eet_application@adp.com
+          overwrite: true
+          attachments:
+            directory: /home/ubuntu/symbits/bose/files/worked_hours/
+          masks:
+            '{search_today}':
+            - today
+            - mask: '%d-%b-%Y'
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ) -> None:
+        self.attachments = {"directory": None}
+        self.search_terms = None
+        super().__init__(
+            loop=loop, job=job, stat=stat, **kwargs
+        )
+
+    async def start(self, **kwargs):  # pylint: disable=W0613
+        if hasattr(self, "attachments"):
+            # attachment directory
+            if "directory" not in self.attachments:
+                self.attachments["directory"] = FILES_PATH.joinpath(
+                    self._program, "files", "download"
+                )
+            else:
+                self.attachments["directory"] = Path(
+                    self.attachments["directory"]
+                ).resolve()
+            try:
+                directory = self.attachments["directory"]
+                if not directory.exists():
+                    if self.create_destination is True:
+                        directory.mkdir(parents=True, exist_ok=True)
+                    else:
+                        raise ComponentError(
+                            f"DownloadFromIMAP: Error creating directory: {directory}"
+                        )
+            except Exception as err:
+                self._logger.error(
+                    f"IMAP: Error creating directory: {err}"
+                )
+                raise ComponentError(
+                    f"IMAP: Error creating directory: {err}"
+                ) from err
+            # masking elements:
+            if hasattr(self, "masks"):
+                for mask, replace in self._mask.items():
+                    for key, val in self.attachments.items():
+                        value = str(val).replace(mask, str(replace))
+                        if isinstance(val, PurePath):
+                            self.attachments[key] = Path(value)
+                        else:
+                            self.attachments[key] = value
+        await super(DownloadFromIMAP, self).start(**kwargs)
+        return True
+
+    async def run(self):
+        self._result = None
+        filter_criteria = []
+        msgs = [""]
+        messages = []
+        files: list = []
+        try:
+            await self.open(self.host, self.port, self.credentials)
+            if not self._client:
+                raise ComponentError("IMAP Connection not Opened, exiting.")
+        except Exception as err:
+            self._logger.exception(err, stack_info=True)
+            raise
+        # getting search Terms
+        self.search_terms = self.process_mask("search_terms")
+        for term, value in self.search_terms.items():
+            try:
+                value = self.mask_replacement(value)
+                filter_criteria.append(f'({term} "{value}")')
+            except (ValueError, KeyError):
+                pass
+        self._logger.debug(f"FILTER CRITERIA: {filter_criteria}")
+        self.add_metric("SEARCH", filter_criteria)
+        try:
+            # getting the Mailbox
+            self._client.select(self.mailbox)
+        except Exception as err:
+            if "User is authenticated but not connected." in str(err):
+                tries = 0
+                while tries < IMAP_RETRY_SELECT:
+                    self._client.logout()
+                    await asyncio.sleep(10)
+                    try:
+                        await self.open(self.host, self.port, self.credentials)
+                        if not self._client:
+                            raise ComponentError(
+                                "IMAP Connection not Opened, exiting."
+                            ) from err
+                        self._client.select(self.mailbox)
+                        break
+                    except Exception as exc:
+                        self._logger.exception(exc, stack_info=True)
+                        if tries < (IMAP_RETRY_SELECT - 1):
+                            tries += 1
+                            continue
+                        else:
+                            raise RuntimeError(
+                                f"IMAP: Error opening Mailbox {self.mailbox}: {exc}"
+                            ) from exc
+        try:
+            print("FILTER ", filter_criteria)
+            result, msgs = self._client.search(None, *filter_criteria)
+            if result == "NO" or result == "BAD":
+                self._logger.error(msgs[0])
+                raise ComponentError(
+                    message=f"Error on Search: {msgs[0]}",
+                    status=500
+                )
+        except imaplib.IMAP4.abort as err:
+            raise ComponentError(f"IMAP Illegal Search: {err}") from err
+        except Exception as err:
+            self._logger.exception(err, stack_info=True)
+            raise ComponentError(f"IMAP Error: {err}") from err
+        msgs = msgs[0].split()
+        i = 0
+        if not msgs:
+            raise FileNotFound("DownloadFromIMAP: File(s) doesn't exists")
+        if not isinstance(msgs, list):
+            raise ComponentError(f"DownloadFromIMAP: Invalid Email Response: {msgs}")
+        if "expected_mime" in self.attachments:
+            expected_mime = self.attachments["expected_mime"]
+            if expected_mime:
+                validate_mime = True
+            else:
+                validate_mime = False
+        else:
+            expected_mime = None
+            validate_mime = False
+        for emailid in msgs:
+            i += 1
+            resp, data = self._client.fetch(emailid.decode("utf-8"), "(RFC822)")
+            if resp == "OK":  # mail was retrieved
+                msg = MailMessage(
+                    self.attachments["directory"],
+                    data[0][1].decode("utf-8"),
+                    data[1],
+                    validate_mime=validate_mime,
+                )
+                messages.append(msg)
+                for attachment in msg.attachments:
+                    if expected_mime is not None:
+                        # checking only for valid MIME types:
+                        if expected_mime != attachment["content_type"]:
+                            continue
+                    if "filename" in self.attachments:
+                        fname = self.attachments[
+                            "filename"
+                        ]  # we only need to save selected files
+                        if not fnmatch.fnmatch(attachment["filename"], fname):
+                            continue
+                        else:
+                            if "rename" in self.attachments:
+                                filename = self.attachments["rename"]
+                                filename = filename.replace(
+                                    "{filename}", Path(attachment["filename"]).stem
+                                )
+                                self._logger.debug(f"NEW FILENAME IS {filename}")
+                                file_path = self.attachments["directory"].joinpath(
+                                    filename
+                                )
+                            else:
+                                file_path = self.attachments["directory"].joinpath(
+                                    attachment["filename"]
+                                )
+                    elif (
+                        "pattern" in self.attachments
+                    ):  # only save files that match the pattern
+                        fpattern = self.attachments["pattern"]
+                        if bool(re.match(fpattern, attachment["filename"])):
+                            file_path = self.attachments["directory"].joinpath(
+                                attachment["filename"]
+                            )
+                        else:
+                            continue
+                    else:
+                        # I need to save everything
+                        if "rename" in self.attachments:
+                            filename = self.attachments["rename"]
+                            if hasattr(self, "masks"):
+                                filename = self.mask_replacement(filename)
+                                # for mask, replace in self._mask.items():
+                                #     filename = filename.replace(mask, replace)
+                            filename = filename.replace(
+                                "{filename}", Path(attachment["filename"]).stem
+                            )
+                            file_path = self.attachments["directory"].joinpath(filename)
+                        else:
+                            file_path = self.attachments["directory"].joinpath(
+                                attachment["filename"]
+                            )
+                    # saving the filename in the attachment
+                    attachment["filename"] = file_path
+                    if "download_existing" in self.attachments:
+                        if (
+                            file_path.exists()
+                            and self.attachments["download_existing"] is False
+                        ):
+                            # we don't need to download again
+                            self._logger.info(f"File Exists {file_path!s}, skipping")
+                            continue
+                    if file_path.exists() and self.overwrite is False:
+                        # TODO: before to save a new renamed file,
+                        #  we need to check if we have it (checksum)
+                        file_name = file_path.name
+                        # dir_name = file_path.absolute()
+                        ext = file_path.suffix
+                        # calculated new filepath
+                        file_path = self.attachments["directory"].joinpath(
+                            f"{file_name}_{i}{ext}"
+                        )
+                        if file_path.exists():
+                            # TODO: more efficient way (while) to check if file exists
+                            files.append(file_path)
+                            continue
+                    await self.save_attachment(file_path, attachment["attachment"])
+                    # saving this file in the list of files:
+                    files.append(file_path)
+            else:
+                raise ComponentError(f"File was not fetch: {resp}")
+        # saving the result:
+        self.add_metric("ATTACHMENTS", files)
+        self.add_metric("NUM_ATTACHMENTS", len(files))
+        self.add_metric('NUM_MESSAGES', len(messages))
+        self._result = {"messages": messages, "files": files}
+        return self._result
+
+    async def save_attachment(self, file_path, content):
+        try:
+            self._logger.info(f"IMAP: Saving attachment file: {file_path}")
+            async with aiofiles.open(file_path, mode="wb") as fp:
+                await fp.write(content)
+        except Exception as err:
+            raise ComponentError(f"File {file_path} was not saved: {err}") from err