flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/FileCopy.py

@@ -0,0 +1,181 @@
+from typing import List
+import asyncio
+import shutil
+import glob
+from pathlib import Path
+from tqdm import tqdm
+from ..exceptions import FileNotFound, FileError, ComponentError
+from .FileBase import FileBase
+from ..utils import check_empty
+from .FileList import FileList
+
+
+class FileCopy(FileBase):
+    """
+    FileCopy
+
+    **Overview**
+
+        Copies files from a source directory to a destination directory.
+
+    **Properties** (inherited from FileBase)
+
+    .. table:: Properties
+        :widths: auto
+
+        +--------------------+----------+-----------+-------------------------------------------------------------------+
+        | Name               | Required | Summary                                                                       |
+        +--------------------+----------+-----------+-------------------------------------------------------------------+
+        | create_destination |   No     | Boolean flag indicating whether to create the destination                     |
+        |                    |          | directory if it doesn't exist (default: True).                                |
+        +--------------------+----------+-----------+-------------------------------------------------------------------+
+        | source             |   Yes    | A dictionary specifying the source directory and filename.                    |
+        |                    |          | (e.g., {"directory": "/path/to/source", "filename": "myfile.txt"})            |
+        +--------------------+----------+-----------+-------------------------------------------------------------------+
+        | destination        |   Yes    | A dictionary specifying the destination directory and optionally filename.    |
+        |                    |          | (e.g., {"directory": "/path/to/destination", "filename": "renamed_file.txt"}) |
+        +--------------------+----------+-----------+-------------------------------------------------------------------+
+        | rename             |   No     | Boolean flag indicating if the file have a new name                           |
+        +--------------------+----------+-----------+-------------------------------------------------------------------+
+        | remove_source      |   No     | Boolean flag indicating whether to remove the source file(s) after            |
+        |                    |          | copying (default: False).                                                     |
+        +--------------------+----------+-----------+-------------------------------------------------------------------+
+
+    
+
+        Example:
+
+        ```yaml
+        FileCopy:
+          source:
+            filename: '*.xlsx'
+            directory: /home/ubuntu/symbits/bose/stores/
+          remove_source: true
+          destination:
+            directory: /home/ubuntu/symbits/bose/stores/backup/
+        ```
+
+    """  # noqa
+
+    progress: bool = True
+
+    async def start(self, **kwargs) -> bool:
+        """Check for File and Directory information."""
+        self._sources: List = []
+        self._source: str = None
+        self._destination: str = None
+        self._rename: bool = False
+        await super(FileCopy, self).start(**kwargs)
+        if hasattr(self, "move"):
+            self.remove_source = self.move
+        elif not hasattr(self, "remove_source"):
+            self.remove_source = False
+        if self.previous and not check_empty(self.input) and (not hasattr(self, "ignore_previous") or self.ignore_previous is False):
+            if isinstance(self.previous, FileList):
+                self._sources = self.input
+                return True
+        if hasattr(self, "source"):
+            # processing source
+            if "directory" not in self.source:
+                raise ComponentError(
+                    "FileCopy: Missing Source Directory."
+                )
+            self._source = Path(self.source["directory"]).resolve()
+            filename = self.source["filename"]
+            if "*" in filename:
+                # is a glob list of files
+                path = self._source.joinpath(filename)
+                listing = glob.glob(str(path))  # TODO using glob from pathlib
+                for fname in listing:
+                    self._logger.debug(f"Filename > {fname}")
+                    self._sources.append(Path(fname))
+            else:
+                filename = self.mask_replacement(filename)
+                path = self._source.joinpath(filename)
+                if path.is_file():
+                    self._logger.debug(f"Source Filename: {filename}")
+                    self._sources.append(path)
+        else:
+            raise ComponentError(
+                "FileCopy: Missing Source information."
+            )
+        if hasattr(self, "destination"):
+            # processing destination
+            if "directory" not in self.destination:
+                raise ComponentError("FileCopy: Missing Destination Directory.")
+            self._destination = Path(self.destination["directory"]).resolve()
+            if not self._destination.exists():  # create the destination directory
+                self._logger.debug(
+                    f"Creating new destination directory: {self._destination}"
+                )
+                try:
+                    self._destination.mkdir(parents=True, exist_ok=True)
+                except OSError as ex:
+                    raise FileError(
+                        f"FileCopy: Error creating destination directory: \
+                        {self._destination}"
+                    ) from ex
+            if "filename" in self.destination:
+                # this is for renaming the file:
+                filename = self.destination["filename"]
+                filename = self.mask_replacement(filename)
+                path = self._destination.joinpath(filename)
+                if self.destination["rename"] is True:
+                    self._rename = True
+                    self._destination = path
+                    if path.is_file():
+                        self._logger.warning(
+                            f"FileCopy: Warning, File {path} already exists,\
+                            will be overwritten"
+                        )
+                else:
+                    self._rename = False
+                    if path.is_file():
+                        self._logger.warning(
+                            f"FileCopy: Warning, File {path} already exists, \
+                            will be overwritten"
+                        )
+            else:
+                filename = None
+                self._rename = False
+        else:
+            raise FileNotFound(
+                "FileCopy: Missing Destination."
+            )
+        return True
+
+    async def run(self):
+        """
+        Delete File(s).
+        """
+        self._result = {}
+        print(f"::::: FileCopy: {self._sources}")
+        if self.progress is True:
+            with tqdm(total=len(self._sources)) as pbar:
+                for file in self._sources:
+                    if self._rename is True:
+                        destination = self._destination
+                    else:
+                        destination = self._destination.joinpath(file.name)
+                    if self.remove_source is True:
+                        await asyncio.to_thread(shutil.move, file, destination)
+                    else:
+                        await asyncio.to_thread(shutil.copyfile, file, destination)
+                    pbar.update(1)
+                    self._result[file.name] = destination
+        else:
+            for file in self._sources:
+                if self._rename is True:
+                    destination = self._destination
+                else:
+                    destination = self._destination.joinpath(file.name)
+                if self.remove_source is True:
+                    await asyncio.to_thread(shutil.move, file, destination)
+                else:
+                    await asyncio.to_thread(shutil.copyfile, file, destination)
+                self._result[file.name] = destination
+        self.add_metric("FILES_COPIED", self._result)
+        return self._result
+
+    async def close(self):
+        """Method."""

flowtask/components/FileDelete.py

@@ -0,0 +1,82 @@
+from tqdm import tqdm
+from ..exceptions import FileError
+from .FileBase import FileBase
+
+
+class FileDelete(FileBase):
+    """
+    FileDelete
+
+
+    Overview
+
+        This component remove all files in a Directory.
+
+    .. table:: Properties
+       :widths: auto
+
+
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | Name         | Required | Summary                                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | file         |   Yes    | A dictionary containing two values, "pattern" and "value",        |
+    |              |          | "pattern" and "value", "pattern" contains the path of the         |
+    |              |          | file on the server, If it contains the mask "{value}", then       |
+    |              |          | "value" is used to set the value of that mask                     |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | pattern      |   Yes    | Allows you to replace values ( ".xls", ".csv", )                  |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | directory    |   Yes    | The directory where are the files to delete                       |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | value        |   Yes    | Name of the function and the arguments it receives for example    |
+    |              |          | [ "current_date", { "mask":"%Y&m%d" } -> 20220909                 |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | dictionary   |   Yes    | Path where to validate if the file exist                          |
+    +--------------+----------+-----------+-------------------------------------------------------+
+
+    
+
+        Example:
+
+        ```yaml
+        FileDelete:
+          file:
+            pattern: '*.csv'
+            value: ''
+          directory: /home/ubuntu/symbits/bayardad/files/job_advertising/bulk/
+        ```
+
+    """
+    progress: bool = True
+
+    async def run(self):
+        """Delete File(s)."""
+        self._result = {}
+        if self.progress is True:
+            filelist = list(self.get_filelist())
+            with tqdm(total=len(filelist)) as pbar:
+                for file in filelist:
+                    # remove all files based on pattern.
+                    try:
+                        file.unlink(missing_ok=True)
+                        self.add_metric("FILE_DELETED", file)
+                        pbar.update(1)
+                    except OSError as err:
+                        raise FileError(
+                            f"FileDelete: Error was raised when delete a File {err}"
+                        ) from err
+        else:
+            filelist = self.get_filelist()
+            for file in filelist:
+                try:
+                    file.unlink(missing_ok=True)
+                    self.add_metric("FILE_DELETED", file)
+                except OSError as err:
+                    raise FileError(
+                        f"FileDelete: Error was raised when delete a File {err}"
+                    ) from err
+        self._result = self.input  # passthroug the previous result.
+        return self._result
+
+    async def close(self):
+        """Method."""

flowtask/components/FileExists.py

@@ -0,0 +1,146 @@
+import logging
+from pathlib import Path, PurePath
+from ..exceptions import FileNotFound, FileError, ComponentError
+from .FileBase import FileBase
+
+
+class FileExists(FileBase):
+    """
+    FileExists
+     Overview
+
+        This component validate if a file or a list of files exist in a directory.
+
+    .. table:: Properties
+       :widths: auto
+
+
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | Name         | Required | Summary                                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | file         |   Yes    | A dictionary containing two values, "pattern" and "value",        |
+    |              |          | "pattern" and "value", "pattern" contains the path of the         |
+    |              |          | file on the server, If it contains the mask "{value}", then       |
+    |              |          | "value" is used to set the value of that mask                     |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | pattern      |   Yes    | Allows you to replace values ( ".xls", ".csv"  )                  |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | value        |   No     | Name of the function and the arguments it receives for example    |
+    |              |          | [ "current_date", { "mask":"%Y&m%d" } -> 20220909                 |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | mask         |   No     | A mask is applied with the today attribute, with date format      |
+    |              |          | {“Y-m-d”}                                                         |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | directory    |   Yes    | Path to validate if the file exists                               |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | date         |   No     | File date                                                         |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | diff         |   No     | Evaluate the difference in the file                               |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | depens       |   No     | The file depends on a previous process                            |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | filename     |   Yes    | Identify the file name                                            |
+    +--------------+----------+-----------+-------------------------------------------------------+
+
+      Return the validation if the file exist or not
+
+    
+
+        Example:
+
+        ```yaml
+        FileExists:
+          file:
+            pattern: '{value}-adp_worker.csv'
+            value:
+            - today
+            - mask: '%Y-%m-%d'
+          directory: /home/ubuntu/symbits/walmart/files/adfs/
+        ```
+
+    """
+
+    async def run(self):
+        """Run File checking."""
+        self._result = {}
+        file = None
+        if isinstance(self._filenames, list) and len(self._filenames) > 0:
+            if hasattr(self, "last_file") or hasattr(self, "first_file"):
+                # only return the last/first file of the list
+                if hasattr(self, "first_file"):
+                    file = self._filenames[0]
+                else:
+                    file = self._filenames[-1]
+                if file.exists() and file.is_file():
+                    self._result[file] = True
+                    self.setTaskVar("DIRECTORY", file.parent)
+                    self.setTaskVar("FILENAME", str(file.name))
+                    self.setTaskVar("FILEPATH", self._path)
+                    if self._debug:
+                        logging.debug(f"fileExists> {file}")
+                else:
+                    raise FileNotFound(f"FileExists: File doesn't exists: {file}")
+            else:
+                # processing a list of files
+                for file in self._filenames:
+                    logging.debug(f" ::: Checking for File: {file}")
+                    if isinstance(file, str):
+                        file = Path(file)
+                    if file.exists() and file.is_file():
+                        self._result[file] = True
+                        if self._debug:
+                            logging.debug(f"fileExists> {file}")
+                        filename = Path(file)
+                        self.setTaskVar("DIRECTORY", filename.parent)
+                        self.setTaskVar("FILENAME", str(filename.name))
+                        self.setTaskVar("FILEPATH", filename)
+                    else:
+                        raise FileNotFound(f"FileExists: File doesn't exists: {file}")
+        elif isinstance(self._filenames, dict):
+            for _, val in self._filenames.items():
+                if "file" in val:
+                    file = val["file"]
+                elif "filename" in val:
+                    file = val["filename"]
+                else:
+                    raise ComponentError(f"FileExists: Wrong Inherit format: {val!r}")
+                if isinstance(file, str):
+                    file = Path(file)
+                elif isinstance(file, PurePath):
+                    file = file.resolve()
+                else:
+                    raise FileError(
+                        f"FileExists: unrecognized type for Filename: {type(file)}"
+                    )
+                logging.debug(f" ::: Checking for File: {file}")
+                if file.exists() and file.is_file():
+                    self._result[file] = True
+                    self.setTaskVar("DIRECTORY", file.parent)
+                    self.setTaskVar("FILENAME", str(file.name))
+                    self.setTaskVar("FILEPATH", file)
+                else:
+                    raise FileNotFound(f"FileExists: Empty result: {file}")
+        elif hasattr(self, "filename"):
+            if isinstance(self.filename, str):
+                file = Path(self.filename)
+            elif isinstance(self.filename, PurePath):
+                file = self.filename.resolve()
+            else:
+                raise FileError(
+                    f"FileExists: unrecognized type for Filename: {type(self.filename)}"
+                )
+            logging.debug(f" ::: Checking for File: {file}")
+            if file.exists() and file.is_file():
+                self._result[file] = True
+                self.setTaskVar("DIRECTORY", file.parent)
+                self.setTaskVar("FILENAME", str(file.name))
+                self.setTaskVar("FILEPATH", file)
+            else:
+                print("RAISE THIS >> ")
+                raise FileNotFound(f"FileExists: Empty result: {file}")
+        # add metric:
+        self.add_metric("FILENAME", self._filenames)
+        return self._result
+
+    async def close(self):
+        """Method."""

flowtask/components/FileIteratorDelete.py

@@ -0,0 +1,112 @@
+import asyncio
+from typing import Any, List
+from collections.abc import Callable
+from pathlib import Path, PurePath
+from tqdm import tqdm
+from ..exceptions import ComponentError
+from .IteratorBase import IteratorBase
+
+
+class FileIteratorDelete(IteratorBase):
+    """
+    FileIteratorDelete
+
+    Overview
+
+        The FileIteratorDelete class is a component for removing all files in a specified directory. It extends the
+        IteratorBase class and provides methods for directory validation, file listing based on patterns, and file deletion.
+
+    .. table:: Properties
+    :widths: auto
+
+        +------------------+----------+--------------------------------------------------------------------------------------------------+
+        | Name             | Required | Description                                                                                      |
+        +------------------+----------+--------------------------------------------------------------------------------------------------+
+        | directory        |   Yes    | The directory from which files will be deleted.                                                  |
+        +------------------+----------+--------------------------------------------------------------------------------------------------+
+        | _filenames       |   No     | A list to store filenames that match the pattern.                                                |
+        +------------------+----------+--------------------------------------------------------------------------------------------------+
+        | _path            |   No     | The Path object for the directory.                                                               |
+        +------------------+----------+--------------------------------------------------------------------------------------------------+
+        | pattern          |   No     | The pattern to match files for deletion.                                                         |
+        +------------------+----------+--------------------------------------------------------------------------------------------------+
+    Return
+
+        The methods in this class manage the deletion of files in a specified directory, including initialization, file listing,
+        and deletion.
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        # self.directory: str = None
+        self._filenames: List[PurePath] = []
+        self.directory: str = None
+        self._path: str = None
+        self.pattern = None
+        super(FileIteratorDelete, self).__init__(
+            loop=loop, job=job, stat=stat, **kwargs
+        )
+
+    async def start(self, **kwargs):
+        """Check if Directory exists."""
+        await super(FileIteratorDelete, self).start()
+        if not self.directory:
+            raise ComponentError(
+                "FileIteratorDelete Error: need to specify a Directory", status=404
+            )
+        # check if directory exists
+        p = Path(self.directory)
+        if p.exists() and p.is_dir():
+            self._path = p
+        else:
+            raise ComponentError(
+                "FileIteratorDelete Error: Directory doesn't exist!", status=404
+            )
+
+    def get_filelist(self) -> List[Any]:
+        if self.pattern:
+            value = self.pattern
+            if hasattr(self, "masks"):
+                for mask, replace in self._mask.items():
+                    value = str(value).replace(mask, replace)
+            if self._variables:
+                value = value.format(**self._variables)
+            files = (f for f in self._path.glob(value))
+        elif hasattr(self, "file"):
+            # using pattern/file version
+            value = self.get_filepattern()
+            files = (f for f in self._path.glob(value))
+        else:
+            files = (f for f in self._path.iterdir() if f.is_file())
+        # files = sorted(files, key=os.path.getmtime)
+        return files
+
+    async def run(self):
+        if not self._path:
+            return False
+        iterator = list(self.get_filelist())
+        with tqdm(total=len(iterator)) as pbar:
+            deleted = []
+            for file in iterator:
+                # remove all files based on pattern.
+                try:
+                    file.unlink(missing_ok=True)
+                    deleted.append(file)
+                    pbar.update(1)
+                except OSError as err:
+                    raise ComponentError(
+                        f"FileIteratorDelete: Error was raised when delete a File {err}"
+                    ) from err
+            self.add_metric("FILES_DELETED", deleted)
+        self._result = self.input  # passthroug the previous result.
+        return self._result
+
+    async def close(self):
+        pass