flowtask 5.8.4__cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/UniqueRows.py

@@ -0,0 +1,126 @@
+import asyncio
+from typing import List
+from collections.abc import Callable
+import pandas as pd
+from asyncdb.exceptions import StatementError, DataError
+from ..exceptions import ComponentError
+from .flow import FlowComponent
+
+
+class UniqueRows(FlowComponent):
+    """
+    UniqueRows
+
+        Overview
+
+            The UniqueRows class is a component for extracting unique rows from a Pandas DataFrame.
+            It supports pre-sorting of rows, custom options for handling duplicates, and an option to save
+            rejected rows that are duplicates.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | unique         |   Yes    | List of columns to use for identifying unique rows.                       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | order          |   No     | Dictionary specifying columns and sort order (`asc` or `desc`).           |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | keep           |   No     | Specifies which duplicates to keep: `first`, `last`, or `False`.          |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | save_rejected  |   No     | Dictionary with filename to save rejected rows as CSV, if specified.      |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component returns a DataFrame containing only unique rows based on the specified columns.
+            If sorting is defined in `order`, rows are pre-sorted before duplicates are removed. Metrics such as
+            the number of rows passed and rejected are recorded. If `save_rejected` is specified, rejected rows
+            are saved to a file. Any data errors encountered during execution are raised with detailed error messages.
+    
+
+        Example:
+
+        ```yaml
+        UniqueRows:
+          unique:
+          - store_id
+        ```
+
+    """  # noqa
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.unique: List = None
+        super(UniqueRows, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Start Component"""
+        if self.previous:
+            self.data = self.input
+
+    def close(self):
+        """Close."""
+
+    async def run(self):
+        """Getting Unique rows from a Dataframe."""
+        self._result = None
+        try:
+            if isinstance(self.data, pd.DataFrame):
+                start = len(self.data.index)
+                self.add_metric("Start", start)
+                if hasattr(self, "order"):
+                    # making a pre-ordering before drop duplicates:
+                    ordering = []
+                    ascending = []
+                    for col, order in self.order.items():
+                        ordering.append(col)
+                        if order == "asc":
+                            ascending.append(True)
+                        else:
+                            order.append(False)
+                    self.data.sort_values(
+                        by=ordering, inplace=True, ascending=ascending
+                    )
+                keep = {}
+                if hasattr(self, "keep"):
+                    keep = {"keep": self.keep}
+                # get only unique rows from this dataframe
+                print('UNIQUE > ', self.unique)
+                self._result = self.data.drop_duplicates(self.unique, **keep)
+                passed = len(self._result.index)
+                self.add_metric("Passed", passed)
+                rejected = start - passed
+                self.add_metric("Rejected", rejected)
+                self._variables[f"{self.StepName}_PASSED"] = passed
+                self._variables[f"{self.StepName}_REJECTED"] = rejected
+                if hasattr(self, "save_rejected"):
+                    # Identify the indices of the rows that were removed
+                    removed_indices = set(self.data.index) - set(self._result.index)
+                    # Select these rows from the original DataFrame
+                    rejected = self.data.loc[list(removed_indices)]
+                    filename = self.save_rejected.get("filename", "rejected_rows.csv")
+                    try:
+                        rejected.to_csv(filename, sep="|")
+                    except IOError:
+                        self._logger.warning(f"Error writing Rejectd File: {filename}")
+                    self.add_metric(
+                        "rejected_file", filename
+                    )
+            else:
+                # return expected data
+                self._result = self.data
+            return self._result
+        except StatementError as err:
+            print(err)
+            return None
+        except DataError as err:
+            print(err)
+            raise ComponentError(f"UniqueRows: Error with Data: {err}") from err

flowtask/components/Unzip.py

@@ -0,0 +1,107 @@
+import os
+import logging
+import asyncio
+from typing import List
+from collections.abc import Callable
+from pathlib import PosixPath, Path, PurePath
+from zipfile import ZipFile, BadZipFile
+from ..exceptions import FileError, ComponentError, FileNotFound
+from .FileCopy import FileCopy
+from ..interfaces.compress import CompressSupport
+
+
+class Unzip(CompressSupport, FileCopy):
+    """
+    Unzip
+
+        Overview
+
+            The Unzip class is a component for decompressing ZIP files in specified directories.
+            It supports selecting specific files within the archive, applying directory masks, and
+            optionally deleting the source ZIP file after extraction.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | filename       |   Yes    | The name of the ZIP file to decompress.                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | directory      |   Yes    | The target directory for decompression.                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | extract        |   No     | Dictionary specifying files to extract and/or target output directory.    |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | delete_source  |   No     | Boolean indicating if the ZIP file should be deleted after extraction.    |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | password       |   No     | Optional password for encrypted ZIP files.                                |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component extracts the specified files from a ZIP archive into the target directory and
+            returns a list of extracted file paths. Metrics such as the output directory and ZIP file name
+            are tracked, and any errors related to file extraction or directory creation are logged for
+            debugging purposes. If specified, the original ZIP file is deleted after extraction.
+    """  # noqa
+
+    _namelist = []
+    _directory = ""
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._output: PurePath = None
+        self._filenames: List = []
+        self.delete_source: bool = False
+        super(Unzip, self).__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+
+    async def start(self, **kwargs):
+        await super().start(**kwargs)
+        return True
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        # Check if File exists
+        self._result = {}
+        for file in self._sources:
+            if not file.exists() or not file.is_file():
+                raise FileNotFound(
+                    f"Compressed File doesn't exists: {file}"
+                )
+            try:
+                files = await self.uncompress_zip(
+                    source=file,
+                    destination=self._destination,
+                    source_files=self._filenames,
+                    password=self.password if hasattr(self, "password") else None,
+                    remove_source=self.delete_source,
+                )
+            except (FileNotFoundError, ComponentError) as err:
+                raise FileError(
+                    f"UnZip failed: {err}"
+                )
+            except RuntimeError as err:
+                raise FileError(
+                    f"UnZip failed: {err}"
+                )
+            if self.delete_source:
+                file.unlink(missing_ok=True)
+
+            filenames = []
+            for filename in files:
+                filenames.append(filename)
+            self._result[str(file)] = filenames
+        return self._result

flowtask/components/UpdateOperationalVars.py

@@ -0,0 +1,147 @@
+"""
+   UpdateOperationalVars
+
+    Overview
+
+        This component updates operational variables stored in the `process_variables` table within the `troc` schema.
+        It supports dynamic variable replacement and mask application, making it versatile for runtime updates in task flows.
+
+    .. table:: Properties
+       :widths: auto
+
+        +--------------+----------+-----------+---------------------------------------------------------------+
+        | Name         | Required | Summary                                                                |
+        +--------------+----------+-----------+---------------------------------------------------------------+
+        | name         |   Yes    | The name of the variable to update.                                     |
+        +--------------+----------+-----------+---------------------------------------------------------------+
+        | value        |   Yes    | The new value to assign to the variable after the update.               |
+        +--------------+----------+-----------+---------------------------------------------------------------+
+        | masks        |   Yes    | Defines a section for dynamically setting the value of the variable.    |
+        +--------------+----------+-----------+---------------------------------------------------------------+
+
+    Returns
+
+        This component does not return data directly. Instead, it updates specified variables in the database and
+        records metrics on the update status. Additionally, logging is provided for SQL execution, and potential errors
+        are managed with detailed logging and exception handling for effective debugging.
+
+
+        Example:
+
+        ```yaml
+        UpdateOperationalVars:
+          name: EPSON_SALES
+          value: '{saturday}'
+          masks:
+            '{saturday}':
+            - date_diff_dow
+            - diff: 2
+              day_of_week: monday
+              mask: '%Y-%m-%d'
+        ```
+
+    """ #noqa
+
+import asyncio
+import logging
+from typing import Any
+from collections.abc import Callable
+from ..exceptions import ComponentError
+from .flow import FlowComponent
+from ..interfaces import DBSupport
+
+
+class UpdateOperationalVars(DBSupport, FlowComponent):
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self.value: Any = None
+        super().__init__(
+            loop=loop, job=job, stat=stat, **kwargs
+        )
+
+    async def start(self, **kwargs):
+        if self.previous:
+            self.data = self.input
+        await super().start(**kwargs)
+        self.processing_credentials()
+        try:
+            self.value = self.mask_replacement(self.value)
+        except Exception as err:
+            raise ComponentError(
+                f"Error adding mask [{err.__class__.__name__}]: {err}"
+            ) from err
+        return True
+
+    async def close(self):
+        pass
+
+    async def set_variable(self, variable, value, program: str = None):
+        if not program:
+            program = self._program
+        sql = f"""UPDATE troc.process_variables SET raw_value = '{value}', \
+        updated_at = CURRENT_DATE
+        WHERE program_slug = '{program}' AND variable_name = '{variable}'"""
+        logging.debug(f"VAR SQL: {sql}")
+        try:
+            connection = self.default_connection('pg')
+        except Exception as err:
+            logging.exception(err, stack_info=True)
+            raise
+        async with await connection.connection() as conn:
+            try:
+                ok = await conn.execute(sql)
+                if ok:
+                    self.add_metric("SET_VARIABLE", f"{variable}={value}")
+                    return True
+                else:
+                    return False
+            except Exception as err:
+                logging.exception(err)
+                return False
+
+    async def run(self):
+        self._result = self.data
+        try:
+            # Replace variables
+            for val in self._variables:
+                if isinstance(self._variables[val], list):
+                    if isinstance(self._variables[val], int):
+                        self._variables[val] = ", ".join(self._variables[val])
+                    else:
+                        self._variables[val] = ", ".join(
+                            "'{}'".format(v) for v in self._variables[val]
+                        )
+                self.value = self.value.replace(
+                    "{{{}}}".format(str(val)), str(self._variables[val])
+                )
+            if hasattr(self, "program_slug"):
+                program = self.program_slug
+            else:
+                program = self._program
+            if hasattr(self, "name"):
+                self._environment.set(self.name, self.value)
+                value = self._environment.get(self.name)
+                if value == self.value:
+                    await self.set_variable(self.name, value, program)
+            elif hasattr(self, "names"):
+                value = None
+                for name in self.names:
+                    try:
+                        self._environment.set(name, self.value)
+                        value = self._environment.get(name)
+                    except Exception as err:
+                        print(err)
+                    if value == self.value:
+                        await self.set_variable(name, value, program)
+            return self._result
+        except Exception as err:
+            print(err)
+            raise ComponentError(
+                f"Error setting operational variable [{err.__class__.__name__}]: {err}"
+            ) from err

flowtask/components/UploadTo.py

@@ -0,0 +1,299 @@
+from typing import List, Dict
+from abc import abstractmethod
+from collections.abc import Callable
+import asyncio
+import logging
+import random
+import ssl
+from pathlib import Path, PosixPath
+from functools import partial
+import aiohttp
+from aiohttp import web
+from tqdm import tqdm
+from datamodel.parsers.json import json_encoder
+from datamodel.parsers.encoders import DefaultEncoder
+from ..exceptions import ComponentError, FileNotFound
+from .flow import FlowComponent
+from ..interfaces.http import ua
+
+
+class UploadToBase(FlowComponent):
+    """
+    UploadToBase
+
+    Overview
+
+        The `UploadToBase` class is an abstract component designed to handle file uploads to various destinations,
+        including servers over HTTP/HTTPS. This class manages credentials, connection settings, SSL configurations,
+        and supports progress tracking during file uploads.
+
+    .. table:: Properties
+    :widths: auto
+
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | Name                    | Required | Description                                                                    |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | url                     |   Yes    | The URL to which files will be uploaded.                                        |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | credentials             |   Yes    | A dictionary containing the credentials necessary for authentication.           |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | source_file             |   No     | The path to the source file to be uploaded.                                     |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | source_dir              |   No     | The directory containing the source files to be uploaded.                       |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | filename                |   No     | The destination filename for the uploaded file.                                 |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | create_destination      |   No     | A flag indicating whether to create the destination directory if it doesn't exist.|
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | ssl                     |   No     | A flag indicating whether to use SSL/TLS for the connection.                    |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | ssl_cafile              |   No     | The path to the CA file for SSL/TLS validation.                                 |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | ssl_certs               |   No     | A list of SSL certificates to be used for the connection.                       |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | host                    |   Yes    | The host address of the destination server.                                     |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | port                    |   Yes    | The port number of the destination server.                                      |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | overwrite               |   No     | A flag indicating whether to overwrite the file if it already exists.           |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | rename                  |   No     | A flag indicating whether to rename the file if a file with the same name exists.|
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | timeout                 |   No     | The timeout value for the upload operation.                                     |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+        | response_status         |   No     | A list of acceptable HTTP response statuses for a successful upload.            |
+        +-------------------------+----------+-----------+----------------------------------------------------------------------------+
+
+    Return
+
+        The methods in this class manage the upload of files to specified destinations, including initialization,
+        execution, and result handling.
+
+    """  # noqa
+    url: str = None
+    _credentials: Dict = {"username": str, "password": str}
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.accept: str = "text/plain"
+        self.url: str = kwargs.get('url', None)
+        self.overwrite: bool = True
+        self.rename: bool = True
+        self.create_destination: bool = kwargs.get('create_destination', False)
+        # source:
+        self.source_file: str = None
+        self.source_dir: str = None
+        # destination:
+        self.filename: str = None
+        self._filenames: List = []
+        self._destination: List = []
+        self._connection: Callable = None
+        self.ssl: bool = False
+        self.ssl_cafile: str = None
+        self.ssl_certs: list = []
+        super().__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        self._encoder = DefaultEncoder()
+        self._valid_response_status: List = (200, 201, 202)
+        # SSL Context:
+        if self.ssl:
+            # TODO: add CAFile and cert-chain
+            self.ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS, cafile=self.ssl_cafile)
+            self.ssl_ctx.options &= ~ssl.OP_NO_SSLv3
+            self.ssl_ctx.verify_mode = ssl.CERT_NONE
+            if self.ssl_certs:
+                self.ssl_ctx.load_cert_chain(*self.ssl_certs)
+        else:
+            self.ssl_ctx = None
+
+    def define_host(self):
+        try:
+            self.host = self.credentials["host"]
+        except KeyError:
+            self.host = self.host
+        try:
+            self.port = self.credentials["port"]
+        except KeyError:
+            self.port = self.port
+        # getting from environment:
+        self.host = self.get_env_value(self.host, default=self.host)
+        self.port = self.get_env_value(str(self.port), default=self.port)
+        if self.host:
+            self._logger.debug(f"<{__name__}>: HOST: {self.host}, PORT: {self.port}")
+
+    def build_headers(self):
+        self.headers = {
+            "Accept": self.accept,
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": random.choice(ua),
+            **self.headers,
+        }
+
+    async def start(self, **kwargs):
+        """Start.
+
+        Processing variables and credentials.
+        """
+        if getattr(self, 'define_host', None) is not None:
+            self.define_host()
+        try:
+            self.processing_credentials()
+        except Exception as err:
+            self._logger.error(err)
+            raise
+        if hasattr(self, "file"):
+            filename = self.process_pattern("file")
+            if hasattr(self, "masks"):
+                filename = self.mask_replacement(filename)
+            # path for file
+            # get path of all files:
+            self._logger.debug("Filename > {}".format(filename))
+            self._filenames.append(filename)
+        if hasattr(self, "source"):  # using the destination filosophy
+            try:
+                if hasattr(self, "masks"):
+                    self.source_dir = self.mask_replacement(self.source["directory"])
+                else:
+                    self.source_dir = self.source["directory"]
+            except KeyError:
+                self.source_dir = "/"
+            print(":: Source Dir: ", self.source_dir)
+            self.source_dir = Path(self.source_dir)
+            # filename:
+            if "file" in self.source:
+                self.source_file = self.process_pattern("file", parent=self.source)
+            else:
+                try:
+                    self.source_file = self.mask_replacement(self.source["filename"])
+                except KeyError:
+                    self.source_file = None
+        if hasattr(self, "destination"):
+            self.directory = self.destination.get('directory')
+            self.directory = self.mask_replacement(self.directory)
+            if isinstance(self.directory, str):
+                self.directory = Path(self.directory).resolve()
+            # Create directory if not exists
+            try:
+                if self.create_destination is True:
+                    self.directory.mkdir(parents=True, exist_ok=True)
+            except OSError as err:
+                raise ComponentError(
+                    f"UploadTo: Error creating destination directory {self.directory}: {err}"
+                ) from err
+            except Exception as err:
+                self._logger.error(
+                    f"Error creating destination directory {self.directory}: {err}"
+                )
+                raise ComponentError(
+                    f"Error creating destination directory {self.directory}: {err}"
+                ) from err
+            try:
+                fname = self.destination.get('filename')
+                if self.directory.exists():
+                    self.filename = self.directory.joinpath(fname)
+                else:
+                    self.filename = fname
+                self._logger.debug(
+                    f"Raw Filename: {self.filename}\n"
+                )
+                if hasattr(self, "masks"):
+                    self.filename = self.mask_replacement(self.filename)
+                    self._destination.append(self.filename)
+            except Exception:
+                pass
+        if self.url:
+            self.build_headers()
+        return True
+
+    async def http_response(self, response: web.Response):
+        """http_response.
+
+        Return the request response of the HTTP Session
+
+        Args:
+            response (web.Response): the Response of the HTTP Session.
+
+        Returns:
+            Any: any processed data.
+        """
+        return response
+
+    async def upload_session(
+        self, url, method: str = "get", data: Dict = None, data_format: str = "json"
+    ):
+        """
+        session.
+            connect to an http source using aiohttp
+        """
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+        # TODO: Auth, Data, etc
+        auth = {}
+        params = {}
+        _data = {"data": None}
+        if self.credentials:
+            if "username" in self.credentials:  # basic Authentication
+                auth = aiohttp.BasicAuth(
+                    self.credentials["username"], self.credentials["password"]
+                )
+                params = {"auth": auth}
+            elif "token" in self.credentials:
+                self.headers["Authorization"] = "{scheme} {token}".format(
+                    scheme=self.credentials["scheme"], token=self.credentials["token"]
+                )
+        if data_format == "json":
+            params["json_serialize"] = json_encoder
+            _data["json"] = data
+        else:
+            _data["data"] = data
+        async with aiohttp.ClientSession(**params) as session:
+            meth = getattr(session, method)
+            if self.ssl:
+                ssl = {"ssl": self.ssl_ctx, "verify_ssl": True}
+            else:
+                ssl = {}
+            fn = partial(
+                meth,
+                self.url,
+                headers=self.headers,
+                timeout=timeout,
+                allow_redirects=True,
+                **ssl,
+                **_data,
+            )
+            try:
+                async with fn() as response:
+                    if response.status in self._valid_response_status:
+                        return await self.http_response(response)
+                    else:
+                        raise ComponentError(
+                            f"UploadTo: Error getting data from URL {response}"
+                        )
+            except aiohttp.HTTPError as err:
+                raise ComponentError(
+                    f"UploadTo: Error Making an SSL Connection to ({self.url}): {err}"
+                ) from err
+            except aiohttp.ClientSSLError as err:
+                raise ComponentError(f"UploadTo: SSL Certificate Error: {err}") from err
+
+    @abstractmethod
+    async def close(self):
+        pass
+
+    @abstractmethod
+    async def run(self):
+        pass
+
+    def start_pbar(self, total: int = 1):
+        return tqdm(total=total)