flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/PGPDecrypt.py

@@ -0,0 +1,199 @@
+import os
+import logging
+import asyncio
+from pathlib import PosixPath, PurePath
+from collections.abc import Callable
+import aiofiles
+import pgpy
+from pgpy.errors import (
+    PGPDecryptionError,
+    PGPOpenSSLCipherNotSupportedError,
+    PGPInsecureCipherError,
+    PGPError,
+)
+from ..exceptions import ComponentError, FileError
+from ..conf import PGP_KEY_PATH, PGP_PASSPHRASE
+from .flow import FlowComponent
+
+
+class PGPDecrypt(FlowComponent):
+    """
+    PGPDecrypt
+
+     Overview
+
+         Decrypt a file encrypted with PGP.
+        TODO: Works with several files (not only one).
+
+    .. table:: Properties
+       :widths: auto
+
+
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | Name         | Required | Summary                                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    |  apply_mask  |   Yes    | This component uses a mask to identify specific bit patterns in a |
+    |              |          | byte of data                                                      |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    |  start       |   Yes    | We initialize the component obtaining the data through the        |
+    |              |          | parameter type                                                    |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    |  close       |   Yes    | The close method of a file object flushes any unwritten data      |
+    |              |          | and closes the file object                                       |
+    +--------------+----------+-----------+-------------------------------------------------------+
+
+
+
+    Return the list of arbitrary days
+
+    
+
+        Example:
+
+        ```yaml
+        PGPDecrypt:
+          filename: LETSTALKMOBILE_NDW_TRANSACTIONS_INFOQUEST_{today}.zip.gpg
+          directory: /home/ubuntu/symbits/xfinity/files/ltm/transactions/
+          decrypt:
+            filename: LETSTALKMOBILE_NDW_TRANSACTIONS_INFOQUEST_{today}.zip
+            directory: /home/ubuntu/symbits/xfinity/files/ltm/transactions/
+          masks:
+            '{today}':
+            - today
+            - mask: '%Y%m%d'
+          delete_source: true
+        ```
+
+    """
+    """
+    PGPDecrypt
+
+
+        Overview
+
+            This component decrypts a PGP encrypted file and saves the decrypted content to a specified location.
+
+        .. table:: Properties
+        :widths: auto
+
+
+        +------------------------+----------+-----------+--------------------------------------------------------------------+
+        | Name                   | Required | Summary                                                                        |
+        +------------------------+----------+-----------+--------------------------------------------------------------------+
+        | filename               |   No     | The name of the file to decrypt.                                               |
+        +------------------------+----------+-----------+--------------------------------------------------------------------+
+        | directory              |   No     | The directory where the encrypted file is located.                             |
+        +------------------------+----------+-----------+--------------------------------------------------------------------+
+        | decrypt                |   No     | Dictionary with "filename" and "directory" keys for output settings.           |
+        +------------------------+----------+-----------+--------------------------------------------------------------------+
+        | delete_source          |   No     | If True, deletes the source encrypted file after decryption. Default is False. |
+        +------------------------+----------+-----------+--------------------------------------------------------------------+
+
+        Returns
+
+        This component returns the path to the decrypted output file.
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.filename: str = None
+        self.directory: PurePath = None
+        self._path: PurePath = None
+        self._output: PurePath = None
+        self.encrypted = None
+        self.key = None
+        self.delete_source: bool = False
+        super(PGPDecrypt, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if isinstance(self.directory, str):
+            self.directory = PosixPath(self.directory).resolve()
+            self._path = PosixPath(self.directory, self.filename)
+        if self.filename is not None:
+            if hasattr(self, "masks"):
+                self.filename = self.mask_replacement(self.filename)
+            self._path = PosixPath(self.directory, self.filename)
+        elif self.previous:
+            if isinstance(self.input, PosixPath):
+                self.filename = self.input
+            elif isinstance(self.input, list):
+                self.filename = self.input[0]
+            elif isinstance(self.input, str):
+                self.filename = PosixPath(self.input)
+            else:
+                filenames = list(self.input.keys())
+                if filenames:
+                    try:
+                        self.filename = filenames[0]
+                    except IndexError as err:
+                        raise FileError("File is empty or doesnt exists") from err
+            self._variables["__FILEPATH__"] = self.filename
+            self._variables["__FILENAME__"] = os.path.basename(self.filename)
+            self._path = self.filename
+        else:
+            raise FileError("File is empty or doesnt exists")
+        if hasattr(self, "decrypt"):
+            if hasattr(self, "masks"):
+                self.decrypt["filename"] = self.mask_replacement(
+                    self.decrypt["filename"]
+                )
+            if "directory" in self.decrypt:
+                PosixPath(self.decrypt["directory"]).mkdir(parents=True, exist_ok=True)
+                self._output = PosixPath(
+                    self.decrypt["directory"], self.decrypt["filename"]
+                )
+            else:
+                self._output = PosixPath(self.directory, self.decrypt["filename"])
+        else:
+            self._output = PosixPath(
+                PosixPath(self._path).parents[0], PosixPath(self._path).stem
+            )
+        self.add_metric("OUTPUT_FILE", self._output)
+        # Encrypted Key
+        try:
+            self.encrypted = pgpy.PGPMessage.from_file(self._path)
+            if PGP_KEY_PATH:
+                self.key, _ = pgpy.PGPKey.from_file(PGP_KEY_PATH)
+            else:
+                raise ComponentError(
+                    "Key path **PGP_KEY_PATH** not defined at environment"
+                )
+        except Exception as err:
+            raise ComponentError(f"{self.__name__} Error: {err}") from err
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        output = None
+        try:
+            with self.key.unlock(PGP_PASSPHRASE):
+                output = self.key.decrypt(self.encrypted).message
+        except (
+            PGPDecryptionError,
+            PGPOpenSSLCipherNotSupportedError,
+            PGPInsecureCipherError,
+            PGPError,
+        ) as err:
+            # Raised when  fails
+            logging.exception(err, stack_info=True)
+            raise ComponentError(
+                f"{err.__class__.__name__}: Error Decripting file: {err}"
+            ) from err
+
+        try:
+            async with aiofiles.open(self._output, "wb") as fp:
+                await fp.write(bytes(output))
+            if self.delete_source is True:
+                self._path.unlink()
+            self._result = self._output
+            return self._result
+        except Exception as err:
+            raise ComponentError(f"{self.__name__} Error: {err}") from err

flowtask/components/PandasIterator.py

@@ -0,0 +1,187 @@
+import asyncio
+from threading import Semaphore
+from decimal import Decimal
+from typing import Any
+from collections.abc import Callable
+import pandas
+import numpy as np
+from navconfig.logging import logging
+from asyncdb.exceptions import NoDataFound, ProviderError
+from ..exceptions import ComponentError, NotSupported, DataNotFound
+from .IteratorBase import IteratorBase, ThreadJob
+
+
+class PandasIterator(IteratorBase):
+    """
+    PandasIterator
+
+        Overview
+
+            This component converts data to a pandas DataFrame in an iterator and processes each row.
+
+        .. table:: Properties
+        :widths: auto
+
+
+        +--------------+----------+-----------+------------------------------------------------------------+
+        | Name         | Required | Summary                                                                |
+        +--------------+----------+-----------+------------------------------------------------------------+
+        | columns      |   Yes    | Names of the columns that we are going to extract.                     |
+        +--------------+----------+-----------+------------------------------------------------------------+
+        | vars         |   Yes    | This attribute organizes names of the columns organized by id.         |
+        +--------------+----------+-----------+------------------------------------------------------------+
+        | parallelize  |   No     | If True, the iterator will process rows in parallel. Default is False. |
+        +--------------+----------+-----------+------------------------------------------------------------+
+        | num_threads  |   No     | Number of threads to use if parallelize is True. Default is 10.        |
+        +--------------+----------+-----------+------------------------------------------------------------+
+
+        Returns
+        -------
+        This component returns the processed pandas DataFrame after iterating
+        through the rows and applying the specified jobs.
+    
+
+        Example:
+
+        ```yaml
+        PandasIterator:
+          columns:
+          - formid
+          - orgid
+          vars:
+            form: '{orgid}/{formid}'
+        ```
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.pk = []
+        self.data = None
+        self._iterator: Any = None
+        self._variables = {}
+        self.vars = {}
+        self._columns = []
+        self._result = None
+        self._num_threads: int = kwargs.pop("num_threads", 10)
+        self._parallelize: bool = kwargs.pop("parallelize", False)
+        super(PandasIterator, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Obtain Pandas Dataframe."""
+        if self.previous:
+            self.data = self.input
+            if isinstance(self.data, pandas.core.frame.DataFrame):
+                if not hasattr(self, "columns"):
+                    # iterate over the total columns of dataframe
+                    self._columns = self.data.columns
+                else:
+                    self._columns = self.columns
+                self._iterator = self.data.iterrows()
+            else:
+                raise ComponentError(
+                    "PandasIterator: Invalid type of Data input (requires a Pandas Dataframe)"
+                )
+        return True
+
+    async def close(self, job=None):
+        close = getattr(job, "close", None)
+        if job:
+            if asyncio.iscoroutinefunction(close):
+                await job.close()
+            else:
+                job.close()
+
+    def createJob(self, target, params, row):
+        """Create the Job Component."""
+        self._result = self.data
+        dt = {}
+        for column in self._columns:
+            value = row[column]
+            if isinstance(value, (int, np.int64, np.integer)):
+                value = int(value)
+            elif isinstance(value, (float, Decimal)):
+                value = float(value)
+            self.setVar(column, value)
+            params[column] = value
+            dt[column] = value
+        args = params["params"]
+        for name, value in self.vars.items():
+            if isinstance(value, list):
+                # TODO: logic to use functions with dataframes
+                # need to calculate the value
+                if name in args:
+                    # replace value with args:
+                    value = args[name]
+                elif name in dt:
+                    value = dt[name]
+            else:
+                if "{" in str(value):
+                    try:
+                        value = value.format(**args)
+                    except KeyError:
+                        value = value.format(**dt)
+            params[name] = value
+            self.setVar(name, value)
+        return self.get_job(target, **params)
+
+    async def run(self):
+        """Async Run Method."""
+        # iterate over next task
+        step, target, params = self.get_step()
+        step_name = step.name
+        i = 0
+        if self._parallelize is True:
+            # parallelized execution
+            threads = []
+            # Limit the number of concurrent threads to 10
+            semaphore = Semaphore(self._num_threads)
+            for _, row in self._iterator:
+                job = self.createJob(target, params, row)
+                if job:
+                    thread = ThreadJob(job, step_name, semaphore)
+                    threads.append(thread)
+                    thread.start()
+            # wait for all threads to finish
+            for thread in threads:
+                thread.join()
+
+                # check if thread raised any exceptions
+                if thread.exc:
+                    raise thread.exc
+        else:
+            for _, row in self._iterator:
+                i += 1
+                # iterate over every row
+                # get I got all values, create a job:
+                job = self.createJob(target, params, row)
+                if job:
+                    try:
+                        self._result = await self.async_job(job, step_name)
+                    except (NoDataFound, DataNotFound) as err:
+                        # its a data component a no data was found
+                        self._logger.notice(
+                            f"Data not Found for Task {step_name}, got: {err}"
+                        )
+                        continue
+                    except (ProviderError, ComponentError) as err:
+                        raise ComponentError(
+                            f"Error on {step_name}, error: {err}"
+                        ) from err
+                    except NotSupported as err:
+                        raise NotSupported(f"Not Supported: {err}") from err
+                    except Exception as err:
+                        raise ComponentError(
+                            f"Component Error {step_name}, error: {err}"
+                        ) from err
+                    finally:
+                        await self.close(job)
+        self._logger.debug(f"Iterations: {i}")
+        self.add_metric("ITERATIONS", i)
+        # returning last value generated by iteration
+        return self._result

flowtask/components/PandasToFile.py

@@ -0,0 +1,189 @@
+import asyncio
+from typing import Dict
+from collections.abc import Callable
+from pathlib import Path
+import csv
+import numpy as np
+import pandas as pd
+from ..exceptions import ComponentError, DataNotFound
+from .flow import FlowComponent
+from ..utils.constants import excel_based
+
+
+class PandasToFile(FlowComponent):
+    """
+    PandasToFile
+
+        Overview
+
+            This component exports a pandas DataFrame to a file in CSV, Excel, or JSON format.
+
+        .. table:: Properties
+        :widths: auto
+
+
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        | Name                   | Required | Summary                                                                     |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        | filename               |   Yes    | The name of the file to save the DataFrame to.                              |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        | directory              |   No     | The directory where the file will be saved. If not specified,               |
+        |                        |          | it will be derived from the filename.                                       |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        | mime                   |   No     | The MIME type of the file. Supported types are "text/csv",                  |
+        |                        |          | "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",        |
+        |                        |          | "application/vnd.ms-excel", "application/json". Default is "text/csv".      |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        | zerofill               |   No     | If True, fills NaN values with "0" in string columns. Default is False.     |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        | quoting                |   No     | Specifies the quoting behavior for CSV files. Options are "all" (QUOTE_ALL),|
+        |                        |          | "string" (QUOTE_NONNUMERIC), and None (QUOTE_NONE). Default is None.        |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        | pd_args                |   No     | Additional arguments for pandas' to_csv, to_excel, or to_json methods.      |
+        |                        |          | Default is an empty dictionary.                                             |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+        |  sep                   |   Yes    | Make a separation of the file name with this sign                           |
+        +------------------------+----------+-----------+-----------------------------------------------------------------+
+
+        Returns
+
+        This component returns the filename of the saved file.
+
+    
+
+        Example:
+
+        ```yaml
+        PandasToFile:
+          filename: /home/ubuntu/symbits/bose/files/report/troc_open_tickets_{today}.csv
+          masks:
+            '{today}':
+            - today
+            - mask: '%Y-%m-%d'
+          mime: text/csv
+          quoting: string
+          pd_args:
+            sep: ','
+        ```
+
+    """  # noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.zerofill: bool = False
+        self.quoting = None
+        self.params: Dict = {}
+        self.args: Dict = {}
+        self.filename: str = None
+        self.directory: str = None
+        self.mime: str = "text/csv"
+        super(PandasToFile, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        if hasattr(self, "pd_args"):
+            self.args = getattr(self, "pd_args", {})
+
+    async def start(self, **kwargs):
+        # Si lo que llega no es un DataFrame de Pandas se cancela la tarea
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError("Data Not Found")
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError(
+                'Incompatible Pandas Dataframe: hint> add "export_dataframe"\
+                :true to tMap component'
+            )
+        if hasattr(self, "masks"):
+            self.filename = self.mask_replacement(self.filename)
+        # Create directory if not exists
+        try:
+            if not self.directory:
+                self.directory = Path(self.filename).parents[0]
+            self.directory.mkdir(parents=True, exist_ok=True)
+        except Exception as err:
+            self._logger.error(
+                f"Error creating directory {self.directory}: {err}"
+            )
+            raise ComponentError(
+                f"Error creating directory {self.directory}: {err}"
+            ) from err
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        self._result = {}
+        if self.zerofill:
+            cols = self.data.select_dtypes(include=["object", "string"])
+            self.data[cols.columns] = cols.fillna("0")
+            # self.data.fillna('0', inplace=True)
+            self.data.replace(np.nan, 0)
+            intcols = self.data.select_dtypes(include=["Int64"])
+            self.data[intcols.columns] = intcols.fillna(0)
+        try:
+            # filename, file_extension = os.path.splitext(self.filename)
+            if self.mime == "text/csv" or self.mime == "text/plain":
+                if self.quoting == "all":
+                    quoting = csv.QUOTE_ALL
+                elif self.quoting == "string":
+                    quoting = csv.QUOTE_NONNUMERIC
+                else:
+                    quoting = csv.QUOTE_NONE
+                #  if file_extension == '.csv':
+                # Los parametros se deben colocar en un diccionario en el JSON
+                # donde las llaves van a ser el nombre de los parametros que se
+                # muestran en la siguiente dirección
+                # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_csv.html
+                if self.data.empty:
+                    raise DataNotFound("PandasToFile: Cannot save an Empty Dataframe.")
+                self._logger.debug(
+                    f"PandasToFile: Export to CSV: {self.filename}"
+                )
+                self.data.to_csv(
+                    self.filename,
+                    index=False,
+                    quoting=quoting,
+                    quotechar='"',
+                    escapechar="\\",
+                    **self.args,
+                )
+            elif self.mime in excel_based:
+                # elif file_extension == '.xlsx':
+                # Los parametros se deben colocar en un diccionario en el JSON
+                # donde las llaves van a ser el nombre de los parametros que se
+                # muestran en la siguiente dirección
+                # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_excel.html
+                self._logger.debug(
+                    f"PandasToFile: Export to EXCEL: {self.filename}"
+                )
+                self.data.to_excel(self.filename, index=False, **self.args)
+            elif self.mime == "application/json":
+                # elif file_extension == '.json':
+                # Los parametros se deben colocar en un diccionario en el JSON
+                # donde las llaves van a ser el nombre de los parametros que se
+                # muestran en la siguiente dirección
+                # https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_json.html
+                self._logger.debug(
+                    f"PandasToFile: Export to JSON: {self.filename}"
+                )
+                self.data.to_json(self.filename, index=False, **self.args)
+            else:
+                raise ComponentError(
+                    "Error: Only extension supported: csv, xlsx and json are supported"
+                )
+            # getting filename:
+            self._result[self.filename] = True
+            self.setTaskVar("FILENAME", self.filename)
+            self.add_metric("FILENAME", self.filename)
+            return self._result
+        except Exception as err:
+            raise ComponentError(f"Error in PandasToFile: {err}") from err