flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/TransposeRows.py

@@ -0,0 +1,176 @@
+import asyncio
+from collections.abc import Callable
+import pandas as pd
+import numpy as np
+from ..exceptions import ComponentError
+from .flow import FlowComponent
+
+
+class TransposeRows(FlowComponent):
+    """
+   TransposeRows
+
+    Overview
+
+        The TransposeRows class is a component for transposing specified rows in a DataFrame by converting row values
+        into new columns based on pivot settings. This component supports options for preserving the original data,
+        handling empty results, and custom column configurations for the transposition.
+
+    .. table:: Properties
+       :widths: auto
+
+        +------------------+----------+-----------+---------------------------------------------------------------+
+        | Name             | Required | Summary                                                                   |
+        +------------------+----------+-----------+---------------------------------------------------------------+
+        | pivot            |   Yes    | List of columns to use as the pivot index for transposition.              |
+        +------------------+----------+-----------+---------------------------------------------------------------+
+        | columns          |   Yes    | Dictionary mapping row values to their target column names.               |
+        +------------------+----------+-----------+---------------------------------------------------------------+
+        | preserve_original|   No     | Boolean indicating if the original rows should be preserved.              |
+        +------------------+----------+-----------+---------------------------------------------------------------+
+        | allow_empty      |   No     | Boolean indicating if empty columns should be allowed in the output.      |
+        +------------------+----------+-----------+---------------------------------------------------------------+
+
+    Returns
+
+        This component returns a DataFrame with specified rows transposed into columns according to the provided pivot
+        and column configurations. If `preserve_original` is set to False, the original rows used in transposition
+        are removed. Any errors in column mapping or pivoting are raised with descriptive error messages.
+    
+
+        Example:
+
+        ```yaml
+        TransposeRows:
+          column: column_name
+          value: data
+          pivot:
+          - formid
+          - form_id
+          - orgid
+          preserve_original: true
+          allow_empty: true
+          columns:
+            '000_001': ad_hoc
+            '000_003': creation_timestamp
+            '000_004': user_device
+            000_008: geoloc
+            000_009: visit_length
+            '000_012': store_name
+            '000_013': store_id
+            '000_023': account_name
+            '000_024': store_designation
+            '000_026': region_name
+            000_029: store_timezone
+            '000_037': visitor_username
+            000_038: visitor_name
+            000_039: visitor_email
+            '000_045': visitor_role
+            '000_055': updated_timestamp
+            '000_065': activity_item_id
+            '000_063': time_in
+            '000_064': time_out
+            '000_066': position_id
+            '000_067': position_manager
+            '000_070': visit_status
+            '194834': retailer
+            VisitDateLocal: visit_timestamp
+        ```
+
+    """ #noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._pivot: list = kwargs.pop("pivot")
+        self.preserve: bool = kwargs.pop("preserve_original", False)
+        self.allow_empty: bool = kwargs.pop("allow_empty", False)
+        if not self._pivot:
+            raise ComponentError("Missing List of Pivot Columns")
+        # columns to be transposed:
+        # TODO: if not, then all columns not in Pivot list.
+        self._columns = kwargs.pop("columns")
+        super(TransposeRows, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+        else:
+            raise ComponentError("Data Not Found", status=404)
+        if not isinstance(self.data, pd.DataFrame):
+            raise ComponentError("Transpose: Incompatible Pandas Dataframe", status=404)
+        if not hasattr(self, "column"):
+            raise ComponentError(
+                "Transpose: Missing Column name for extracting row values"
+            )
+        if not hasattr(self, "value"):
+            raise ComponentError("Transpose: Missing Column for Values")
+        return True
+
+    async def close(self):
+        pass
+
+    def row_to_column(self, df: pd.DataFrame, row_to_pivot: str, new_name: str):
+        """
+        Add a pivoted column to the dataframe based on the given column name.
+
+        Parameters:
+        - df: The input dataframe.
+        - row_to_pivot: The column name to be pivoted.
+        - new_name: The name of the column to be transposed.
+
+        Returns:
+        - Dataframe with the new pivoted column.
+        """
+        # Filter the dataframe to only include rows with the desired column_name
+        df_filtered = df[df[self.column] == row_to_pivot]
+        if df_filtered.empty is True:
+            # there is no data to be filtered:
+            if self.allow_empty is True:
+                df[new_name] = np.nan
+            return df
+        # Pivot the filtered dataframe
+        df_pivot = df_filtered.pivot_table(
+            index=self._pivot,
+            columns=self.column,
+            values=self.value,
+            aggfunc="first",
+            dropna=False,  # Preserve NaN values
+        ).reset_index()
+        df_pivot = df_pivot.rename(columns={row_to_pivot: new_name})
+        # Merge the pivoted dataframe with the original dataframe
+        df_merged = pd.merge(df, df_pivot, on=self._pivot, how="left")
+        if self.preserve is False:
+            # Drop the original column_name and value columns for the pivoted rows
+            df_merged = df_merged.drop(
+                df_merged[(df_merged[self.column] == row_to_pivot)].index
+            )
+        return df_merged
+
+    async def run(self):
+        try:
+            df = self.data
+            for column, value in self._columns.items():
+                try:
+                    df_pivot = self.row_to_column(df, column, value)
+                except Exception as err:
+                    print(err)
+                df = df_pivot
+            if self._debug is True:
+                print("=== TRANSPOSE ===")
+                print(" = Data Types: = ")
+                print(df.dtypes)
+                print("::: Printing Column Information === ")
+                for column, t in df.dtypes.items():
+                    print(column, "->", t, "->", df[column].iloc[0])
+            self._result = df
+            return self._result
+        except (ValueError, KeyError) as err:
+            raise ComponentError(f"Crosstab Error: {err!s}") from err
+        except Exception as err:
+            raise ComponentError(f"Unknown error {err!s}") from err

flowtask/components/UPCDatabase.py

@@ -0,0 +1,86 @@
+from datetime import date
+from .RESTClient import AbstractREST
+
+
+class UPCDatabase(AbstractREST):
+    """
+    UPCDatabase.
+
+    Querying UPC Database for Product information.
+    
+
+        Example:
+
+        ```yaml
+        UPCDatabase:
+          method: currency
+          base: USD
+          credentials:
+            apikey: UPC_API_KEY
+        ```
+
+    """
+
+    base_url: str = "https://api.upcdatabase.org/"
+    _default_method: str = "currency"
+
+    async def start(self, **kwargs):
+        if not self.credentials:
+            # obtengo las credenciales
+            self.credentials["apikey"] = self._environment.get("UPC_API_KEY")
+            if not self.credentials["apikey"]:
+                raise ValueError("UPC Database: Missing Credentials")
+        # calling method for this element:
+        await super(UPCDatabase, self).start(**kwargs)
+
+    async def currency(self):
+        """currency.
+
+        Currency information and exchange rates supported by UPC
+        """
+        self.method = "get"
+        self.url = self.base_url + "currency/latest/"
+        try:
+            self.parameters["base"] = self.base
+        except (ValueError, TypeError):
+            self.parameters["base"] = "USD"
+
+    async def currency_history(self):
+        """currency.
+
+        Retrieves the currency conversion rates for a specific date.
+        """
+        self.method = "get"
+        self.url = self.base_url + "currency/history"
+        try:
+            self.parameters["base"] = self.base
+        except (ValueError, TypeError):
+            self.parameters["base"] = "USD"
+        try:
+            self.parameters["date"] = self.date
+        except (ValueError, AttributeError):
+            today = date.today()
+            self.parameters["date"] = today.strftime("%Y-%m-%d")
+
+    async def product(self):
+        """product.
+
+        Product information based on UPC barcode
+        """
+        self.method = "get"
+        if not "barcode" in self._args:
+            raise ValueError("UPC Database: Missing Barcode")
+        self.url = self.base_url + "product/{barcode}"
+
+    async def search(self):
+        """product.
+
+        Search for a product based on item parameters.
+        """
+        self.method = "get"
+        self.url = self.base_url + "search/"
+        self.parameters["page"] = 1
+        try:
+            self.parameters["query"] = self.query
+        except (ValueError, TypeError) as e:
+            raise ValueError("UPC Database: Missing or wrong *query* Search.") from e

flowtask/components/UnGzip.py

@@ -0,0 +1,171 @@
+import os
+import logging
+import asyncio
+from typing import List
+from collections.abc import Callable
+from pathlib import PosixPath, Path, PurePath
+import tarfile
+from ..exceptions import FileError, ComponentError, FileNotFound
+from .FileCopy import FileCopy
+from ..utils import check_empty
+from ..interfaces.compress import CompressSupport
+
+
+class UnGzip(CompressSupport, FileCopy):
+    """
+    UnGzip
+
+        Overview
+
+            The UnGzip class is a component for decompressing Gzip (.gz) files, including compressed tarballs (e.g., .tar.gz, .tar.bz2, .tar.xz).
+            This component extracts the specified Gzip or tarball file into a target directory and supports optional source file deletion
+            after extraction.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | filename       |   Yes    | The path to the Gzip file to uncompress.                                  |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | directory      |   Yes    | The target directory where files will be extracted.                       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | delete_source  |   No     | Boolean indicating if the source file should be deleted post-extraction.  |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | extract        |   No     | Dictionary specifying filenames to extract and/or output directory.       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component extracts files from a specified Gzip or tarball archive into the designated directory
+            and returns a list of paths to the extracted files. It tracks metrics for the output directory and the source
+            Gzip file. If configured, the original compressed file is deleted after extraction. Errors encountered during
+            extraction or directory creation are logged and raised as exceptions.
+    
+
+        Example:
+
+        ```yaml
+        UnGzip:
+          source:
+            directory: /home/ubuntu/symbits/mso/files/commissions_statements/pr/
+            filename: STATEMENT_STATEMENT-*.CSV.gz
+          destination:
+            directory: /home/ubuntu/symbits/mso/files/commissions_statements/pr/
+          delete_source: true
+        ```
+
+    """  # noqa
+
+    _namelist = []
+    _directory = ""
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._output: PurePath = None
+        self._filenames: List = []
+        self.delete_source: bool = False
+        self._as_list: bool = kwargs.pop('as_list', False)
+        self._binary_sources: dict = None
+        super(UnGzip, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if self.previous and not check_empty(self.input) and getattr(self, "ignore_previous", False) is False:
+            if isinstance(self.input, dict):
+                self._binary_sources = self.input.get('files', {})
+                return True
+        await super().start(**kwargs)
+        # Handle extraction settings
+        if hasattr(self, "destination"):
+            # New format using destination
+            if isinstance(self.destination, dict) and "directory" in self.destination:
+                self._destination = Path(self.destination["directory"]).resolve()
+            else:
+                self._destination = Path(self.destination).resolve()
+        elif hasattr(self, "extract"):
+            # Legacy format using extract
+            for _, filename in enumerate(self.extract["filenames"]):
+                filename = self.mask_replacement(filename)
+                self._filenames.append(filename)
+            if "directory" in self.extract:
+                self._destination = Path(self.extract["directory"]).resolve()
+            else:
+                # same directory as source
+                self._destination = Path(self.directory)
+        else:
+            # If no destination specified, use source directory
+            self._destination = Path(self.directory)
+        # Create destination directory if it doesn't exist
+        try:
+            self._destination.mkdir(parents=True, exist_ok=True)
+        except Exception as err:
+            logging.error(f"Error creating directory {self._destination}: {err}")
+            raise ComponentError(
+                f"Error creating directory {self._destination}: {err}"
+            ) from err
+        # For backwards compatibility
+        self._output = self._destination
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        # Check if file exists
+        self._result = {}
+        filenames = []
+        if self._binary_sources:
+            for filename, contents in self._binary_sources.items():
+                # get binary contents:
+                data = contents.get('data')
+                data.seek(0)  # restarts from beginning
+                # Uncompress gzip contents in memory:
+                try:
+                    f = await self.uncompress_gzip(
+                        source=data,
+                        destination=None
+                    )
+                    filenames.append(
+                        {"filename": filename, "data": f}
+                    )
+                except RuntimeError as err:
+                    raise FileError(f"UnGzip failed: {err}")
+            self._result = filenames
+        elif self._sources:
+            for file in self._sources:
+                if not file.exists() or not file.is_file():
+                    raise FileNotFound(
+                        f"Compressed File doesn't exist: {file}"
+                    )
+                # Uncompress the gzip/tar.gz file
+                try:
+                    files = await self.uncompress_gzip(
+                        source=file,
+                        destination=self._destination,
+                        remove_source=self.delete_source,
+                    )
+                except (FileNotFoundError, ComponentError) as err:
+                    raise FileError(f"UnGzip failed: {err}")
+                except RuntimeError as err:
+                    raise FileError(f"UnGzip failed: {err}")
+
+                if self.delete_source:
+                    file.unlink(missing_ok=True)
+
+                filenames = []
+                for filename in files:
+                    filenames.append(filename)
+                self._result[str(file)] = filenames
+            if self._as_list is True:
+                self._result = filenames
+            self.add_metric("OUTPUT_DIRECTORY", self._destination)
+        self.add_metric("FILES_UNCOMPRESS", len(filenames))
+        # Support both metrics for backwards compatibility
+        return self._result

flowtask/components/Uncompress.py

@@ -0,0 +1,172 @@
+import os
+import logging
+import asyncio
+from typing import List
+from collections.abc import Callable
+from pathlib import PosixPath, Path, PurePath
+from zipfile import ZipFile, BadZipFile
+from ..exceptions import FileError, ComponentError, FileNotFound
+from .flow import FlowComponent
+from ..interfaces.compress import CompressSupport
+
+
+class Uncompress(CompressSupport, FlowComponent):
+    """
+    Uncompress
+
+        Overview
+
+            The Uncompress class is a component for decompressing files in various archive formats, including but not limited to:
+            7z (.7z), ACE (.ace), ALZIP (.alz), AR (.a), ARC (.arc), ARJ (.arj), BZIP2 (.bz2), CAB (.cab), compress (.Z), 
+            CPIO (.cpio), DEB (.deb), DMS (.dms), GZIP (.gz), LRZIP (.lrz), LZH (.lha, .lzh), LZIP (.lz), LZMA (.lzma), 
+            LZOP (.lzo), RPM (.rpm), RAR (.rar), RZIP (.rz), TAR (.tar), XZ (.xz), ZIP (.zip, .jar), and ZOO (.zoo). 
+            It extracts the specified compressed file into a target directory and can optionally delete the source file 
+            upon successful extraction.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | filename       |   Yes    | The path to the compressed file to be decompressed.                       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | directory      |   Yes    | The target directory where files will be extracted.                       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | delete_source  |   No     | Boolean indicating if the source file should be deleted post-extraction.  |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | extract        |   No     | Dictionary specifying filenames to extract and/or output directory.       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | password       |   No     | Optional password for encrypted files in supported formats.               |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component extracts files from the specified compressed archive into the designated directory and returns
+            a list of paths to the extracted files. It tracks metrics for the output directory and compressed file name.
+            If configured, the original compressed file is deleted after extraction. Errors related to file corruption 
+            or extraction issues are logged and raised as exceptions.
+    
+
+        Example:
+
+        ```yaml
+        Uncompress:
+          filename: organizational_unit_ancestors_{yesterday}.zip
+          masks:
+            yesterday:
+            - yesterday
+            - mask: '%Y-%m-%d'
+          directory: /home/ubuntu/symbits/polestar/files/organizational_unit_ancestors/
+          extract:
+            filenames:
+            - OrganizationalUnitAncestors.csv
+            directory: /home/ubuntu/symbits/polestar/files/organizational_unit_ancestors/
+          delete_source: false
+        ```
+
+    """ #noqa
+
+    _namelist = []
+    _directory = ""
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.filename: str = None
+        self.directory: PurePath = None
+        self._path: PurePath = None
+        self._output: PurePath = None
+        self._filenames: List = []
+        self.delete_source: bool = False
+        super(Uncompress, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        if isinstance(self.directory, str):
+            self.directory = PosixPath(self.directory).resolve()
+            self._path = PosixPath(self.directory, self.filename)
+        if self.filename is not None:
+            if hasattr(self, "masks"):
+                self.filename = self.mask_replacement(self.filename)
+            self._path = PosixPath(self.directory, self.filename)
+        elif self.previous:
+            if isinstance(self.input, PosixPath):
+                self.filename = self.input
+            elif isinstance(self.input, list):
+                self.filename = PosixPath(self.input[0])
+            elif isinstance(self.input, str):
+                self.filename = PosixPath(self.input)
+            else:
+                filenames = list(self.input.keys())
+                if filenames:
+                    try:
+                        self.filename = PosixPath(filenames[0])
+                    except IndexError as err:
+                        raise FileError("File is empty or doesnt exists") from err
+            self._variables["__FILEPATH__"] = self.filename
+            self._variables["__FILENAME__"] = os.path.basename(self.filename)
+            self._path = self.filename
+        else:
+            raise FileError("UnCompress: File is empty or doesn't exists")
+        if hasattr(self, "extract"):
+            for _, filename in enumerate(self.extract["filenames"]):
+                filename = self.mask_replacement(filename)
+                self._filenames.append(filename)
+            if "directory" in self.extract:
+                self._output = Path(self.extract["directory"]).resolve()
+                # Create directory if not exists
+                try:
+                    self._output.mkdir(parents=True, exist_ok=True)
+                except Exception as err:
+                    logging.error(f"Error creating directory {self._output}: {err}")
+                    raise ComponentError(
+                        f"Error creating directory {self._output}: {err}"
+                    ) from err
+            else:
+                # same directory:
+                self._output = Path(self.directory)
+        self.add_metric("OUTPUT_DIRECTORY", self._output)
+        self.add_metric("ZIP_FILE", self.filename)
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        # Check if File exists
+        self._result = None
+        if not self._path.exists() or not self._path.is_file():
+            raise FileNotFound(
+                f"Compressed File doesn't exists: {self._path}"
+            )
+        with ZipFile(self._path) as zfp:
+            status = zfp.testzip()
+            if not status:  # Si no hay status
+                # If not exists namelist, extract all files
+                members = self._filenames if self._filenames else None
+                # If not exists password return None
+                pwd = self.password if hasattr(self, "password") else None
+                try:
+                    zfp.extractall(path=self._output, members=members, pwd=pwd)
+                    if self.delete_source is True:
+                        self._path.unlink()
+                except BadZipFile as err:
+                    # The error raised for bad ZIP files.
+                    raise ComponentError(f"Bad Zip File: {err}") from err
+                except Exception as err:
+                    # Undefined error
+                    raise ComponentError(f"{self.__name__} ZIP Error: {err}") from err
+                result = self._filenames if self._filenames else zfp.namelist()
+                filenames = []
+                for filename in result:
+                    f = self._output.joinpath(filename)
+                    filenames.append(f)
+                self._result = filenames
+            else:  # Si hay status devuelve el primer archivo corrupto
+                raise ComponentError(f"Zip File {status} is corrupted")
+        return self._result