flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/FileList.py

@@ -0,0 +1,194 @@
+import os
+import asyncio
+import logging
+from collections.abc import Callable
+from pathlib import Path
+from tqdm import tqdm
+from threading import Semaphore
+from asyncdb.exceptions import ProviderError
+from ..exceptions import ComponentError, NotSupported, FileNotFound
+from ..utils import check_empty
+from .IteratorBase import IteratorBase, ThreadJob
+
+
+class FileList(IteratorBase):
+    """
+    FileList with optional Parallelization Support.
+
+    Overview
+
+    This component iterates through a specified directory and returns a list of files based on a provided pattern or individual files.
+    It supports asynchronous processing and offers options for managing empty results and detailed error handling.
+
+
+    .. table:: Properties
+    :widths: auto
+
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | Name                | Required | Description                                                                                           |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | directory (str)     |   Yes    | Path to the directory containing the files to be listed.                                              |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | pattern (str)       |    No    | Optional glob pattern for filtering files (overrides individual files if provided).                   |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | filename (str)      |    No    | Name of the files                                                                                     |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | iterate (bool)      |    No    | Flag indicating whether to iterate through the files and process them sequentially (defaults to True).|
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | generator (bool)    |    No    | Flag controlling the output format: `True` returns a generator, `False` (default) returns a list.     |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | file (dict)         |    No    | A dictionary containing two values, "pattern" and "value", "pattern" and "value",                     |
+    |                     |          | "pattern" contains the path of the file on the server, If it contains the mask "{value}",             |
+    |                     |          | then "value" is used to set the value of that mask                                                    |
+    +---------------------+----------+-----------+-------------------------------------------------------------------------------------------+
+    | parallelize         |    No    | If True, the iterator will process rows in parallel. Default is False.                                |
+    +---------------------+----------+-----------+-------------------------------------------------------------------------------------------+
+    | num_threads         |    No    | Number of threads to use if parallelize is True. Default is 10.                                       |
+    +---------------------+----------+-----------+-------------------------------------------------------------------------------------------+
+
+    Return the list of files in a Directory
+    
+
+        Example:
+
+        ```yaml
+        FileList:
+          directory: /home/ubuntu/symbits/bayardad/files/job_advertising/bulk/
+          pattern: '*.csv'
+          iterate: true
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.generator: bool = False
+        self._path = None
+        self.pattern = None
+        self.data = None
+        self.directory: str = None
+        self._num_threads: int = kwargs.pop("num_threads", 10)
+        self._parallelize: bool = kwargs.pop("parallelize", False)
+        super(FileList, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Check if Directory exists."""
+        await super(FileList, self).start()
+        if not self.directory:
+            raise ComponentError("Error: need to specify a Directory")
+        if isinstance(self.directory, str) and "{" in self.directory:
+            self.directory = self.mask_replacement(self.directory)
+            print(f"Directory: {self.directory}")
+        # check if directory exists
+        p = Path(self.directory)
+        if p.exists() and p.is_dir():
+            self._path = p
+        else:
+            raise ComponentError("Error: Directory doesn't exist!")
+        return True
+
+    def get_filelist(self):
+        if self.pattern:
+            value = self.pattern
+            if "{" in value:
+                value = self.mask_replacement(value)
+            if self._variables:
+                value = value.format(**self._variables)
+            files = (f for f in self._path.glob(value))
+        elif hasattr(self, "file"):
+            # using pattern/file version
+            value = self.get_filepattern()
+            files = (f for f in self._path.glob(value) if f.is_file())
+        else:
+            files = (f for f in self._path.iterdir() if f.is_file())
+        files = sorted(files, key=os.path.getmtime)
+        return files
+
+    async def run(self):
+        status = False
+        if not self._path:
+            return False
+        if self.iterate:
+            files = list(self.get_filelist())
+            step, target, params = self.get_step()
+            step_name = step.name
+            if self._parallelize:
+                # Parallelized execution
+                threads = []
+                semaphore = Semaphore(self._num_threads)
+                with tqdm(total=len(files)) as pbar:
+                    for file in files:
+                        self._result = file
+                        params["filename"] = file
+                        params["directory"] = self.directory
+                        job = self.get_job(target, **params)
+                        if job:
+                            pbar.set_description(f"Processing {file.name}")
+                            thread = ThreadJob(job, step_name, semaphore)
+                            threads.append(thread)
+                            thread.start()
+                    # wait for all threads to finish
+                    results = []
+                    for thread in threads:
+                        thread.join()
+                        # check if thread raised any exceptions
+                        if thread.exc:
+                            raise thread.exc
+                        pbar.update(1)
+                        results.append(thread.result)
+                if check_empty(results):
+                    return False
+                else:
+                    self._result = results
+                    return self._result
+            else:
+                # generate and iterator
+                with tqdm(total=len(files)) as pbar:
+                    for file in files:
+                        self._result = file
+                        params["filename"] = file
+                        params["directory"] = self.directory
+                        logging.debug(f" :: Loading File: {file}")
+                        status = False
+                        job = self.get_job(target, **params)
+                        if job:
+                            pbar.set_description(f"Processing {file.name}")
+                            try:
+                                status = await self.async_job(job, step_name)
+                                pbar.update(1)
+                            except (ProviderError, ComponentError, NotSupported) as err:
+                                raise NotSupported(
+                                    f"Error running Component {step_name}, error: {err}"
+                                ) from err
+                            except Exception as err:
+                                raise ComponentError(
+                                    f"Generic Component Error on {step_name}, error: {err}"
+                                ) from err
+                if check_empty(status):
+                    return False
+                else:
+                    return status
+        else:
+            files = self.get_filelist()
+            if files:
+                if self.generator is False:
+                    self._result = list(files)
+                else:
+                    self._result = files
+                if len(self._result) < 100:
+                    self.add_metric("FILE_LIST", self._result)
+                self.add_metric(
+                    "FILE_LIST_COUNT", len(self._result)
+                )
+                return self._result
+            else:
+                raise FileNotFound(f"FileList: No files found {files}")
+
+    async def close(self):
+        pass

flowtask/components/FileOpen.py

@@ -0,0 +1,75 @@
+from pathlib import Path, PurePath
+import io
+import aiofiles
+from ..exceptions import FileNotFound, FileError
+
+from .FileBase import FileBase
+
+
+class FileOpen(FileBase):
+    """
+    FileOpen.
+
+        **Overview**
+
+        This component opens one or more files asynchronously and returns their contents as streams.
+        It supports handling both individual filenames and lists of filenames.
+        It provides error handling for missing files or invalid file types.
+
+
+        .. table:: Properties
+        :widths: auto
+
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | Name                | Required | Description                                                                                           |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | directory (str)     |   Yes    | Path to the directory containing the files to be listed.                                              |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | pattern (str)       |    No    | Optional glob pattern for filtering files (overrides individual files if provided).                   |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | filename (str)      |    No    | Name of the files                                                                                     |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | file (dict)         |    No    | A dictionary containing two values, "pattern" and "value", "pattern" and "value",                     |
+    |                     |          | "pattern" contains the path of the file on the server, If it contains the mask "{value}",             |
+    |                     |          | then "value" is used to set the value of that mask                                                    |
+    +---------------------+----------+-----------+-------------------------------------------------------------------------------------------+
+
+    """  # noqa
+
+    async def run(self):
+        """Run File checking."""
+        self._result = {}
+        file = None
+        if isinstance(self._filenames, list) and len(self._filenames) > 0:
+            # concatenate all files in one result:
+            for file in self._filenames:
+                if isinstance(file, str):
+                    file = Path(file)
+                if file.exists() and file.is_file():
+                    async with aiofiles.open(file, "r+") as afp:
+                        content = await afp.read()
+                        stream = io.StringIO(content)
+                        stream.seek(0)
+                        self._result[file.name] = stream
+                else:
+                    self._logger.error(f"FileExists: File doesn't exists: {file}")
+        elif hasattr(self, "filename"):
+            if isinstance(self.filename, str):
+                file = Path(file)
+            elif isinstance(self.filename, PurePath):
+                file = self.filename.resolve()
+            else:
+                raise FileError(
+                    f"FileExists: unrecognized type for Filename: {type(self.filename)}"
+                )
+            if file.exists() and file.is_file():
+                async with aiofiles.open(file, "r+") as afp:
+                    content = await afp.read()
+                    stream = io.StringIO(content)
+                    stream.seek(0)
+                    self._result[file.name] = stream
+            else:
+                raise FileNotFound(f"FileExists: Empty result: {self._filenames}")
+        # add metric:
+        self.add_metric("FILENAME", self._filenames)
+        return self._result

flowtask/components/FileRead.py

@@ -0,0 +1,120 @@
+from collections.abc import Callable
+import asyncio
+from pathlib import Path, PurePath
+import io
+import aiofiles
+from ..exceptions import FileNotFound, FileError
+from ..utils.json import json_decoder
+from .FileBase import FileBase
+from ..interfaces.dataframes.pandas import PandasDataframe
+
+
+class FileRead(FileBase, PandasDataframe):
+    """
+    FileRead.
+
+        **Overview**
+
+        Read an String File and returned as string (non-binary)
+
+
+        .. table:: Properties
+        :widths: auto
+
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | Name                | Required | Description                                                                                           |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | directory (str)     |   Yes    | Path to the directory containing the files to be listed.                                              |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | pattern (str)       |    No    | Optional glob pattern for filtering files (overrides individual files if provided).                   |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | filename (str)      |    No    | Name of the files                                                                                     |
+    +---------------------+----------+-------------------------------------------------------------------------------------------------------+
+    | file (dict)         |    No    | A dictionary containing two values, "pattern" and "value", "pattern" and "value",                     |
+    |                     |          | "pattern" contains the path of the file on the server, If it contains the mask "{value}",             |
+    |                     |          | then "value" is used to set the value of that mask                                                    |
+    +---------------------+----------+-----------+-------------------------------------------------------------------------------------------+
+
+
+
+        Example:
+
+        ```yaml
+        FileRead:
+          file: recap_response_payloads.json
+          directory: recaps/
+          is_json: true
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self._use_taskstorage = kwargs.get('use_taskstorage', False)
+        super(FileRead, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def close(self):
+        """Method."""
+
+    async def start(self, **kwargs):
+        """Check for File and Directory information."""
+        if not hasattr(self, "directory"):
+            if self._use_taskstorage:
+                self.directory = self._taskstore.get_path().joinpath(self._program, 'files')
+            else:
+                self.directory = self._filestore.get_directory('')
+        await super().start(**kwargs)
+        return True
+
+    async def run(self):
+        """Run File checking."""
+        self._result = {}
+        file = None
+        result = ""
+        if isinstance(self._filenames, list) and len(self._filenames) > 0:
+            # concatenate all files in one result:
+            for file in self._filenames:
+                if isinstance(file, str):
+                    file = Path(file)
+                if file.exists() and file.is_file():
+                    async with aiofiles.open(file, "r+") as afp:
+                        content = await afp.read()
+                        result += content
+                else:
+                    self._logger.error(
+                        f"FileExists: File doesn't exists: {file}"
+                    )
+        elif hasattr(self, "filename"):
+            if isinstance(self.filename, str):
+                file = Path(file)
+            elif isinstance(self.filename, PurePath):
+                file = self.filename.resolve()
+            else:
+                raise FileError(
+                    f"FileExists: unrecognized type for Filename: {type(self.filename)}"
+                )
+            if file.exists() and file.is_file():
+                async with aiofiles.open(file, "r+") as afp:
+                    content = await afp.read()
+                    result = result + content
+            else:
+                raise FileNotFound(f"FileExists: Empty result: {self._filenames}")
+        # add metric:
+        if hasattr(self, 'is_json'):
+            try:
+                result = json_decoder(result)
+            except Exception as exc:
+                self._logger.error(
+                    f":: Error decoding JSON: {exc}"
+                )
+                return None
+        if getattr(self, 'as_dataframe', False):
+            result = await self.create_dataframe(result)
+        self._result = result
+        self.add_metric("FILENAMES", self._filenames)
+        return self._result

flowtask/components/FileRename.py

@@ -0,0 +1,106 @@
+import os
+import asyncio
+from pathlib import Path
+from ..exceptions import FileNotFound, FileError, ComponentError
+from .FileBase import FileBase
+
+
+class FileRename(FileBase):
+    """
+    FileRename.
+
+    **Overview**
+
+    This component renames a file asynchronously based on provided source and destination paths.
+    It supports handling missing files and offers optional behavior for ignoring missing source files.
+    It performs basic validation to ensure the destination doesn't overwrite existing files.
+
+
+    .. table:: Properties
+    :widths: auto
+
+    +--------------------------+----------+---------------------------------------------------------------------------------------------------+
+    | Name                     | Required | Description                                                                                       |
+    +--------------------------+----------+---------------------------------------------------------------------------------------------------+
+    | directory (str)          |   Yes    | Path to the directory containing the source file.                                                 |
+    +--------------------------+----------+-----------+---------------------------------------------------------------------------------------+
+    | destination (str)        |   Yes    | New filename (with optional variable replacement using `set_variables` and `mask_replacement`)    |
+    |                          |          | for the file.                                                                                     |
+    +--------------------------+----------+-----------+---------------------------------------------------------------------------------------+
+    | ignore_missing (bool)    |    No    | Flag indicating whether to ignore missing source files (defaults to False).                       |
+    +--------------------------+----------+---------------------------------------------------------------------------------------------------+
+    | source (str)             |   Yes    | Filename (with optional variable replacement using `set_variables` and `mask_replacement`)        |
+    |                          |          | of the file to rename.                                                                            |
+    +--------------------------+----------+-----------+---------------------------------------------------------------------------------------+
+
+
+    
+
+        Example:
+
+        ```yaml
+        FileRename:
+          Group: ICIMSRename
+          ignore_missing: true
+          directory: /home/ubuntu/symbits/icims/files/forms/
+          source: '{form_id}_{form_data_id}.txt'
+          destination: '{form_id}_{form_data_id}_{associate_id} - {full_name}.txt'
+        ```
+
+    """  # noqa
+
+    async def start(self, **kwargs) -> bool:
+        """Check for File and Directory information."""
+        self._source: str = None
+        self._destination: str = None
+        if not hasattr(self, "ignore_missing"):
+            self.ignore_missing = False
+        if not hasattr(self, "directory"):
+            raise ComponentError("Missing Source Directory.")
+        if isinstance(self.directory, str):
+            self.directory = Path(self.directory).resolve()
+        if hasattr(self, "source"):
+            # processing source
+            filename = self.set_variables(self.mask_replacement(self.source))
+            self._logger.notice(f"Source File {filename}")
+            path = self.directory.joinpath(filename)
+            if "*" in filename:
+                raise ComponentError(
+                    "FileRename: Cannot Support wildcard on filenames."
+                )
+            else:
+                if path.is_file():
+                    self._logger.debug(f"Source Filename: {filename}")
+                    self._source = path
+                else:
+                    if self.ignore_missing is False:
+                        raise FileNotFound(f"File {path} was not found.")
+        else:
+            raise ComponentError("FileRename: Missing Source information.")
+        if hasattr(self, "destination"):
+            # processing destination
+            filename = self.set_variables(self.mask_replacement(self.destination))
+            path = self.directory.joinpath(filename)
+            if (
+                self._source and path.exists()
+            ):  # we cannot rename a file overwriting another.
+                raise FileError(f"Cannot Rename to {filename}, file Exists")
+            self._destination = path
+        else:
+            raise FileNotFound("FileRename: Missing Destination.")
+        await super(FileRename, self).start(**kwargs)
+        return True
+
+    async def run(self):
+        """Delete File(s)."""
+        self._result = {}
+        if self._source is not None and self._source.exists():
+            await asyncio.to_thread(os.rename, self._source, self._destination)
+            self._result[self._source] = self._destination
+            self.add_metric("FILE_RENAMED", self._result)
+        if self.ignore_missing is False:
+            raise FileNotFound(f"Source File {self._source} was not found.")
+        return self._result
+
+    async def close(self):
+        """Method."""