flowtask 5.8.4__cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/components/UploadToS3.py

@@ -0,0 +1,136 @@
+import os
+import asyncio
+import logging
+from collections.abc import Callable
+import aiofiles
+from ..exceptions import FileError
+from .UploadTo import UploadToBase
+from ..interfaces.Boto3Client import Boto3Client
+
+
+class UploadToS3(Boto3Client, UploadToBase):
+    """
+    UploadToS3
+
+    Overview
+
+        The `UploadToS3` class is a specialized component that facilitates the uploading of files to an Amazon S3 bucket. 
+        This class extends both the `Boto3Client` and `UploadToBase` classes, providing integration with AWS S3 for file 
+        storage operations. The component supports the upload of individual files, multiple files, or entire directories.
+
+    .. table:: Properties
+    :widths: auto
+
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | Name                    | Required | Description                                                                  |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | bucket                  |   Yes    | The name of the S3 bucket to which files will be uploaded.                   |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | directory               |   Yes    | The S3 directory path where files will be uploaded.                         |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | source_dir              |   Yes    | The local directory containing the files to be uploaded.                    |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | _filenames              |   Yes    | A list of filenames to be uploaded.                                         |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | whole_dir               |   No     | A flag indicating whether to upload all files in the source directory.       |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | ContentType             |   No     | The MIME type of the files to be uploaded. Defaults to "binary/octet-stream".|
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+        | credentials             |   Yes    | A dictionary containing the credentials necessary for AWS authentication.    |
+        +-------------------------+----------+-----------+--------------------------------------------------------------------------+
+
+    Return
+
+        The `run` method uploads files to the specified S3 bucket, returning a dictionary containing the list of successfully 
+        uploaded files and any errors encountered during the upload process.
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.mdate = None
+        self.local_name = None
+        self.filename: str = ""
+        self.whole_dir: bool = False
+        self.preserve = True
+        self.ContentType: str = "binary/octet-stream"
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """start Method."""
+        await super(UploadToS3, self).start(**kwargs)
+        if hasattr(self, "destination"):
+            self.directory = self.destination["directory"]
+            if not self.directory.endswith("/"):
+                self.directory = self.source_dir + "/"
+            self.directory = self.mask_replacement(self.destination["directory"])
+        if hasattr(self, "source"):
+            self.whole_dir = (
+                self.source["whole_dir"] if "whole_dir" in self.source else False
+            )
+            if self.whole_dir is True:
+                # if whole dir, is all files in source directory
+                logging.debug(f"Uploading all files on directory {self.source_dir}")
+                p = self.source_dir.glob("**/*")
+                self._filenames = [x for x in p if x.is_file()]
+            else:
+                if "filename" in self.source:
+                    p = self.source_dir.glob(self.filename)
+                    self._filenames = [x for x in p if x.is_file()]
+        try:
+            if self.previous and self.input:
+                self._filenames = self.input
+            if hasattr(self, "file"):
+                filenames = []
+                for f in self._filenames:
+                    p = self.source_dir.glob(f)
+                    fp = [x for x in p if x.is_file()]
+                    filenames = filenames + fp
+                self._filenames = filenames
+        except (NameError, KeyError):
+            pass
+        return self
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        """Running Upload file to S3."""
+        self._result = None
+        try:
+            use_credentials = self.credentials["use_credentials"]
+            # del self.credentials['use_credentials']
+        except KeyError:
+            use_credentials = False
+        async with self.get_client(
+            use_credentials, credentials=self.credentials, service=self.service
+        ) as s3_client:
+            errors = {}
+            files = {}
+            for file in self._filenames:
+                key = os.path.basename(file)
+                filename = f"{self.directory}{key}"
+                print("FILENAME ", filename)
+                # TODO: making async with chunks (part data)
+                async with aiofiles.open(file, mode="rb") as f:
+                    content = await f.read()
+                    response = await s3_client.put_object(
+                        Bucket=self.bucket,
+                        Key=filename,
+                        Body=content,
+                        ContentType=self.ContentType,
+                    )
+                    rsp = response["ResponseMetadata"]
+                    status_code = int(rsp["HTTPStatusCode"])
+                    if status_code == 200:
+                        files[file] = filename
+                    else:
+                        errors[file] = FileError(f"S3: Upload Error: {rsp!s}")
+            self._result = {"files": files, "errors": errors}
+            self.add_metric("S3_UPLOADED", files)
+            return self._result

flowtask/components/UploadToSFTP.py

@@ -0,0 +1,160 @@
+import asyncio
+from collections.abc import Callable
+import asyncssh
+from ..exceptions import ComponentError, FileNotFound
+from .UploadTo import UploadToBase
+from ..interfaces.SSHClient import SSHClient
+
+
+class UploadToSFTP(SSHClient, UploadToBase):
+    """
+    UploadToSFTP
+
+        Overview
+
+            The UploadToSFTP class is a component for uploading files or entire directories to an SSH/SFTP server.
+            It supports various configurations, including recursive directory uploads, customizable transfer settings,
+            and real-time upload progress tracking.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | source         |   Yes    | A dictionary specifying the source directory, filename, and/or            |
+            |                |          | recursive setting for selecting files to upload.                          |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | destination    |   Yes    | A dictionary defining the target directory on the SFTP server.            |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | whole_dir      |   No     | Boolean indicating if the entire source directory should be uploaded.     |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | block_size     |   No     | Integer defining the block size for file transfer, defaults to 65356.     |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | max_requests   |   No     | Integer setting the max number of parallel requests, defaults to 1.       |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component uploads files to the specified SFTP directory and returns a list of uploaded files on success.
+            If no files are found or a connection error occurs, it raises a relevant exception. Metrics on the number of
+            files uploade
+
+
+        Example:
+
+        ```yaml
+        UploadToSFTP:
+          host: sftp.example.com
+          port: 22
+          credentials:
+            username: sftpuser
+            password: abcd1234
+          destination:
+            directory: /incoming/
+        ```
+
+    """ #noqa
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.mdate = None
+        self.local_name = None
+        self.filename: str = ""
+        self.whole_dir: bool = False
+        self.preserve = True
+        self.block_size: int = 65356
+        self.max_requests: int = 1
+        super().__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """start Method."""
+        await super(UploadToSFTP, self).start(**kwargs)
+        if hasattr(self, "source"):
+            self.whole_dir = (
+                self.source["whole_dir"] if "whole_dir" in self.source else False
+            )
+        if hasattr(self, "destination"):
+            self.directory = self.destination["directory"]
+        try:
+            if self.previous and self.input:
+                self.filename = self.input
+            elif self.file:
+                self.filename = self.process_pattern("file")
+        except (NameError, KeyError):
+            pass
+        return self
+
+    def upload_progress(self, srcpath, dstpath, bytes_copied, total_bytes):
+        self._pb.reset(total=total_bytes)
+        self._pb.update(bytes_copied)
+        self._pb.refresh()
+
+    async def run(self):
+        """Running Download file."""
+        self._result = None
+        status = False
+        try:
+            async with await self.open(
+                host=self.host,
+                port=self.port,
+                tunnel=self.tunnel,
+                credentials=self.credentials,
+            ):
+                async with self._connection.start_sftp_client() as sftp:
+                    # check all versions of functionalities
+                    args = {
+                        "block_size": self.block_size,
+                        "max_requests": self.max_requests,
+                        "progress_handler": self.upload_progress,
+                        "error_handler": self.err_handler,
+                    }
+                    if self.whole_dir is True:
+                        self._logger.debug(
+                            f"Uploading all files on directory {self.source_dir}"
+                        )
+                        file = "{}/*".format(self.source_dir)
+                        p = self.source_dir.glob("**/*")
+                        self.filename = [x for x in p if x.is_file()]
+                    else:
+                        file = self.filename
+                    args["remotepath"] = self.directory
+                    if hasattr(self, "source"):
+                        args["recurse"] = True if "recursive" in self.source else False
+                    if not file:
+                        raise FileNotFound(f"There is no local File: {file}")
+                    self.start_progress(total=len(file))
+                    try:
+                        self._logger.debug(f"Uploading file: {file} to {self.directory}")
+                        status = await sftp.mput(file, **args)
+                        self.close_progress()
+                    except OSError as err:
+                        self._logger.error(f"Upload SFTP local error: {err}")
+                        return False
+                    except asyncssh.sftp.SFTPError as err:
+                        self._logger.error(f"SFTP UploadTo Server error: {err}")
+                        return False
+                    except asyncssh.Error as err:
+                        self._logger.error(f"Upload 2 SFTP: connection failed: {err}")
+                    except Exception as err:
+                        self._logger.exception(err)
+                        raise
+        except asyncio.CancelledError:
+            self._logger.info(
+                f"{self.host} CANCELED~"
+            )
+            # break
+        except (asyncio.TimeoutError, ComponentError) as err:
+            raise ComponentError(f"{err!s}") from err
+        except Exception as err:
+            raise ComponentError(f"{err!s}") from err
+        if status is False:
+            return False
+        else:
+            self.add_metric("SFTP_FILES", self.filename)
+            self._result = self.filename
+            return self._result

flowtask/components/UploadToSharepoint.py

@@ -0,0 +1,205 @@
+import os
+from collections.abc import Callable
+import asyncio
+import logging
+from pathlib import Path
+from ..exceptions import (
+    FileError,
+    ConfigError
+)
+from .UploadTo import UploadToBase
+from ..interfaces.Sharepoint import SharepointClient
+
+
+class UploadToSharepoint(SharepointClient, UploadToBase):
+    """
+    UploadToSharepoint
+
+        Overview
+
+            The UploadToSharepoint class is a component for uploading files or entire directories to a SharePoint site.
+            It supports various configuration options for selecting files by name, extension, or pattern, and includes
+            functionality for recursive directory searches.
+
+        .. table:: Properties
+        :widths: auto
+
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | Name           | Required | Summary                                                                   |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | credentials    |   Yes    | A dictionary with SharePoint credentials: `username`, `password`,         |
+            |                |          | `tenant`, and `site`.                                                     |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | source         |   Yes    | A dictionary specifying the source directory, filename, and/or file       |
+            |                |          | extension for selecting files to upload.                                  |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | destination    |   Yes    | A dictionary defining the SharePoint destination directory.               |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | whole_dir      |   No     | Boolean indicating if the entire source directory should be uploaded.     |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+            | recursive      |   No     | Boolean specifying whether to search recursively within directories.      |
+            +----------------+----------+-----------+---------------------------------------------------------------+
+
+        Returns
+
+            This component uploads files to a specified SharePoint directory and returns the upload result status.
+            It logs the upload activity and records metrics for the number of files uploaded. If no files are found
+            or the configuration is incomplete, it raises an error. The upload can handle both individual files and
+            entire folders, depending on configuration.
+    
+
+        Example:
+
+        ```yaml
+        UploadToSharepoint:
+          credentials:
+            username: sharepoint_username
+            password: sharepoint_password
+            tenant: symbits
+            site: trocstorage
+          destination:
+            directory: Shared Documents/Optimum Sales Files
+          masks:
+            '{today}':
+            - today
+            - mask: '%Y%m%d%H%M%S'
+        ```
+
+    """  # noqa
+
+    # dict of expected credentials
+    _credentials: dict = {
+        "username": str,
+        "password": str,
+        "tenant": str,
+        "site": str
+    }
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        self.mdate = None
+        self.local_name = None
+        self.filename: str = ""
+        self.whole_dir: bool = False
+        self.preserve = True
+        self.ContentType: str = "binary/octet-stream"
+        self.recursive: bool = kwargs.get('recursive', False)
+        super().__init__(
+            loop=loop,
+            job=job,
+            stat=stat,
+            **kwargs
+        )
+        self.define_host = None
+
+    async def start(self, **kwargs):
+        """start Method."""
+        await super(UploadToSharepoint, self).start(**kwargs)
+        if hasattr(self, "source"):
+            self.source_dir = self.source.get('directory')
+            if isinstance(self.source_dir, str):
+                self.source_dir = Path(self.source_dir).resolve()
+            self.filename = self.source.get("filename", None)
+            self.whole_dir = (
+                self.source["whole_dir"] if "whole_dir" in self.source else False
+            )
+            if self.whole_dir is True:
+                # if whole dir, is all files in source directory
+                logging.debug(
+                    f"Uploading all files on directory {self.source_dir}"
+                )
+                p = self.source_dir.glob("**/*")
+                self._filenames = [
+                    x for x in p if x.is_file()
+                ]
+            else:
+                if "filename" in self.source:
+                    filename = self.source_dir.joinpath(self.source_file)
+                    if filename.is_dir():
+                        p = self.source_dir.glob(filename)
+                        self._filenames = [
+                            x for x in p if x.is_file()
+                        ]
+                    else:
+                        self._filenames = [filename]
+                elif 'extension' in self.source:
+                    extension = self.source["extension"]
+                    pattern = self.source.get("pattern", None)
+                    if pattern:
+                        # TODO: fix recursive problem from Glob
+                        if self.recursive is True:
+                            p = self.source_dir.rglob(f"**/*{pattern}*{extension}")
+                        else:
+                            p = self.source_dir.glob(f"*{pattern}*{extension}")
+                    else:
+                        if self.recursive is True:
+                            p = self.source_dir.rglob(f"**/*{extension}")
+                        else:
+                            p = self.source_dir.glob(f"*{pattern}*{extension}")
+                    self._filenames = [
+                        x for x in p if x.is_file()
+                    ]
+                else:
+                    raise ConfigError(
+                        "UploadToSharepoint: No filename or extension in source"
+                    )
+        if hasattr(self, "destination"):
+            # Destination in Sharepoint:
+            self.directory = self.destination["directory"]
+            if not self.directory.endswith("/"):
+                self.directory = str(self.directory) + "/"
+            self.directory = self.mask_replacement(self.destination["directory"])
+        else:
+            if self.previous and self.input:
+                self._filenames = self.input
+            if hasattr(self, "file"):
+                filenames = []
+                for f in self._filenames:
+                    p = self.source_dir.glob(f)
+                    fp = [x for x in p if x.is_file()]
+                    filenames = filenames + fp
+                self._filenames = filenames
+            elif 'extension' in self.source:
+                extension = self.source["extension"]
+                pattern = self.source.get("pattern", None)
+                # check if files in self._filenames ends with extension
+                filenames = []
+                for f in self._filenames:
+                    if f.suffix == extension:
+                        # check if pattern is in the filename
+                        if pattern:
+                            if pattern in f.name:
+                                filenames.append(f)
+                            continue
+                        else:
+                            filenames.append(f)
+                self._filenames = filenames
+        return self
+
+    async def close(self):
+        pass
+
+    async def run(self):
+        """Upload a File to Sharepoint"""
+        self._result = None
+        async with self.connection():
+            if not self.context:
+                self.context = self.get_context(self.url)
+        if not self._filenames:
+            raise FileError("No files to upload")
+        if self.whole_dir is True:
+            # Using Upload entire Folder:
+            self._result = await self.upload_folder(
+                local_folder=self.source_dir
+            )
+        else:
+            self._result = await self.upload_files(
+                filenames=self._filenames
+            )
+        self.add_metric("SHAREPOINT_UPLOADED", self._result)
+        return self._result

flowtask/components/UserFunc.py

@@ -0,0 +1,122 @@
+from typing import Union
+import importlib
+import asyncio
+import logging
+from collections.abc import Callable, Awaitable
+from ..exceptions import ComponentError
+from settings.settings import TASK_STORAGES
+from .flow import FlowComponent
+
+
+def getFunction(program, function):
+    """getFunction.
+
+        Example:
+
+        ```yaml
+        UserFunc:
+          function: scheduling_visits
+          args:
+            max_distance: 400
+            max_stores: 5
+            year: 2024
+            month: 11
+        ```
+
+    """
+    ## TODO: detect TaskStorage of the task
+    storage = TASK_STORAGES["default"]
+    fn_path = storage.path.joinpath(program, "functions", f"{function}.py")
+    try:
+        spec = importlib.util.spec_from_file_location(function, fn_path)
+        module = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(module)
+        obj = getattr(module, function)
+        return obj
+    except ImportError as e:
+        logging.error(f"UserFunc: No Function {function} was Found")
+        raise ComponentError(
+            f"UserFunc: No Python Function {function} was Found on {fn_path}"
+        ) from e
+
+
+class UserFunc(FlowComponent):
+    """
+    UserFunc.
+
+       Overview
+
+       Run a arbitrary user function and return result
+
+     .. table:: Properties
+       :widths: auto
+
+
+    +--------------+----------+-----------+-------------------------------------------------------+
+    | Name         | Required | Summary                                                           |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    |  function    |   Yes    | Name function                                                     |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    |  params      |   Yes    | Allows you to set parameters                                      |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    |  foo         |   Yes    | Variable name                                                     |
+    +--------------+----------+-----------+-------------------------------------------------------+
+    |  api_keys    |   Yes    | Api password to query                                             |
+    +--------------+----------+-----------+-------------------------------------------------------+
+
+    Return the list of arbitrary days
+
+
+
+
+    """
+
+    def __init__(
+        self,
+        loop: asyncio.AbstractEventLoop = None,
+        job: Callable = None,
+        stat: Callable = None,
+        **kwargs,
+    ):
+        """Init Method."""
+        self._fn: Union[Callable, Awaitable] = None
+        self.data = None
+        self.params = None
+        self.function: Callable = None
+        self._kwargs = kwargs.get('args', {})
+        super(UserFunc, self).__init__(loop=loop, job=job, stat=stat, **kwargs)
+
+    async def start(self, **kwargs):
+        """Obtain Previous data."""
+        if self.previous:
+            self.data = self.input
+        try:
+            self._fn = getFunction(self._program, self.function)
+        except ComponentError as err:
+            raise ComponentError(
+                f"UserFunc: Error getting Function from {self.function}"
+            ) from err
+
+    async def close(self):
+        """Close Method."""
+
+    async def run(self):
+        """Run Method."""
+        self._result = None
+        params = {"data": self.data, "variables": self._variables, **self._kwargs}
+        if self.params:
+            params = {**params, **self.params}
+        try:
+            if asyncio.iscoroutinefunction(self._fn):
+                result = await self._fn(
+                    self, loop=self._loop, env=self._environment, **params
+                )
+            else:
+                result = self._fn(
+                    self, loop=self._loop, env=self._environment, **params
+                )
+            self._result = result
+            self.add_metric("UDF", f"{self._fn!s}")
+            return self._result
+        except ComponentError as err:
+            raise ComponentError(f"UserFunc: Error calling {self._fn}: {err}") from err