flowtask 5.8.4__cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl

flowtask/interfaces/smartsheet.py

@@ -0,0 +1,154 @@
+"""
+SmartSheet.
+
+Making operations over an Smartsheet Service.
+
+"""
+import random
+from functools import partial
+from pathlib import Path
+import ssl
+import aiofiles
+import aiohttp
+from ..exceptions import (
+    ComponentError,
+    ConfigError,
+    FileNotFound
+)
+from .credentials import CredentialsInterface
+from ..interfaces.http import ua
+
+
+class SmartSheetClient(CredentialsInterface):
+    _credentials: dict = {"token": str, "scheme": str}
+
+    def __init__(self, *args, **kwargs):
+        self.file_format: str = "application/vnd.ms-excel"
+        self.url: str = "https://api.smartsheet.com/2.0/sheets/"
+        self.create_destination: bool = True  # by default
+        self.file_id: str = kwargs.pop('file_id', None)
+        api_key = self.get_env_value("SMARTSHEET_API_KEY")
+        self.api_key: str = kwargs.pop('api_key', api_key)
+        kwargs['no_host'] = True
+        super().__init__(*args, **kwargs)
+        if not self.api_key:
+            raise ComponentError(
+                f"SmartSheet: Invalid API Key name {self.api_key}"
+            )
+        self.ssl_certs = kwargs.get('ssl_certs', [])
+        self.ssl_ctx = ssl.SSLContext(ssl.PROTOCOL_TLS)
+        self.ssl_ctx.options &= ~ssl.OP_NO_SSLv3
+        self.ssl_ctx.minimum_version = ssl.TLSVersion.TLSv1_2
+        self.ssl_ctx.check_hostname = False
+        self.ssl_ctx.verify_mode = ssl.CERT_NONE
+        if self.ssl_certs:
+            self.ssl_ctx.load_cert_chain(*self.ssl_certs)
+
+    async def http_get(
+        self,
+        url: str = None,
+        credentials: dict = None,
+        headers: dict = {},
+        accept: str = 'application/vnd.ms-excel',
+        destination: Path = None
+    ):
+        """
+        session.
+            connect to an http source using aiohttp
+        """
+        timeout = aiohttp.ClientTimeout(total=self.timeout)
+        if url is None:
+            url = self.url
+        # TODO: Auth, Data, etc
+        auth = {}
+        params = {}
+        headers = {
+            "Accept": accept,
+            "Accept-Encoding": "gzip, deflate",
+            "DNT": "1",
+            "Connection": "keep-alive",
+            "Upgrade-Insecure-Requests": "1",
+            "User-Agent": random.choice(ua),
+            **headers,
+        }
+        if credentials:
+            if "username" in credentials:  # basic Authentication
+                auth = aiohttp.BasicAuth(
+                    credentials["username"], credentials["password"]
+                )
+                params = {"auth": auth}
+            elif "token" in credentials:
+                headers["Authorization"] = "{scheme} {token}".format(
+                    scheme=credentials["scheme"], token=credentials["token"]
+                )
+        async with aiohttp.ClientSession(**params) as session:
+            meth = getattr(session, 'get')
+            ssl = {"ssl": self.ssl_ctx, "verify_ssl": True}
+            fn = partial(
+                meth,
+                url,
+                headers=headers,
+                timeout=timeout,
+                allow_redirects=True,
+                **ssl,
+            )
+            try:
+                async with fn() as response:
+                    if response.status in (200, 201, 202):
+                        return await self.http_response(response, destination)
+                    else:
+                        print("ERROR RESPONSE >> ", response)
+                        raise ComponentError(
+                            f"Smartsheet: Error getting data from URL {response}"
+                        )
+            except Exception as err:
+                raise ComponentError(
+                    f"Smartsheet: Error Making an SSL Connection to ({self.url}): {err}"
+                ) from err
+            except aiohttp.exceptions.HTTPError as err:
+                raise ComponentError(
+                    f"Smartsheet: SSL Certificate Error: {err}"
+                ) from err
+
+    async def http_response(self, response, destination: str):
+        # getting aiohttp response:
+        if response.status == 200:
+            try:
+                async with aiofiles.open(str(destination), mode="wb") as fp:
+                    await fp.write(await response.read())
+                if destination.exists():
+                    return response
+                else:
+                    raise FileNotFound(
+                        f"Error saving File {destination!s}"
+                    )
+            except FileNotFound:
+                raise
+            except Exception as err:
+                raise FileNotFound(
+                    f"Error saving File {err!s}"
+                )
+        else:
+            raise ComponentError(
+                f"DownloadFromSmartSheet: Wrong response from Smartsheet: {response!s}"
+            )
+
+    async def download_file(self, file_id: str = None, destination: Path = None):
+        if isinstance(destination, str):
+            destination = Path(destination)
+        if not file_id:
+            file_id = self.file_id
+        if not file_id:
+            raise ConfigError(
+                "SmartSheet: Unable to Download without FileId."
+            )
+        credentials = {"token": self.api_key, "scheme": "Bearer"}
+        url = f"{self.url}{file_id}"
+        if not destination:
+            destination = self.filename
+        if await self.http_get(
+            url=url,
+            credentials=credentials,
+            destination=destination
+        ):
+            return True

flowtask/interfaces/stat.py

@@ -0,0 +1,39 @@
+from abc import ABC
+from typing import Optional, ParamSpec, Any
+import traceback
+from ..utils.stats import TaskMonitor
+
+P = ParamSpec("P")
+
+class StatSupport(ABC):
+    """StatSupport.
+
+    Adding Support for Task Monitor (Statistics Collector.)
+    """
+
+    def __init__(
+        self,
+        *args: P.args,
+        **kwargs: P.kwargs
+    ):
+        # stats object:
+        self._stat_: bool = True
+        stat: Optional[TaskMonitor] = kwargs.pop('stat', None)
+        if stat:
+            self.stat: Optional[TaskMonitor] = stat
+        else:
+            self.stat = None
+            self._stat_ = False
+        super().__init__(*args, **kwargs)
+
+    def save_traceback(self):
+        try:
+            self.stat.stacktrace(traceback.format_exc())
+        finally:
+            pass
+
+    def add_metric(self, name, value):
+        try:
+            self.stat.add_metric(name, value)
+        except AttributeError:
+            pass

flowtask/interfaces/task.py

@@ -0,0 +1,96 @@
+"""
+Flowtask Task Execution.
+"""
+from typing import Optional
+from abc import ABC
+import asyncio
+from navconfig.logging import logging
+# Queue Worker Client:
+from qw.client import QClient
+from qw.wrappers import TaskWrapper
+# TODO: Dispatch tasks to docker container.
+from ..tasks.task import Task
+from ..exceptions import FlowTaskError, TaskFailed
+
+
+class TaskSupport(ABC):
+    def __init__(self, *args, **kwargs):
+        self._name_ = self.__class__.__name__
+        self.program: str = kwargs.pop("program", "navigator")
+        self.task: str = kwargs.pop('task')
+        # No worker:
+        self._no_worker: bool = kwargs.pop("no_worker", False)
+        if self._no_worker is False:
+            self.worker: QClient = QClient()  # auto-discovering of workers
+        else:
+            self.worker: Optional[QClient] = None
+        # self.worker = worker
+        self.priority = kwargs.pop("priority", None)
+        self.logger = logging.getLogger(
+            f"Task.{self.program}.{self.task}"
+        )
+        self._args = args
+        self._kwargs = kwargs
+        super().__init__(*args, **kwargs)
+
+    def __repr__(self):
+        return f"<Action.Task>: {self.program}.{self.task}"
+
+    async def run(self, *args, **kwargs):
+        if self._no_worker is False:
+            if self.priority == "pub":
+                # Using Channel Group mechanism (avoid queueing)
+                try:
+                    result = await self.worker.publish(self.wrapper)
+                    await asyncio.sleep(0.01)
+                    return result
+                except asyncio.TimeoutError:
+                    raise
+                except Exception as exc:
+                    self.logger.error(f"{exc}")
+                    raise
+            else:
+                try:
+                    result = await self.worker.queue(self.wrapper)
+                    await asyncio.sleep(0.01)
+                    return result
+                except asyncio.TimeoutError:
+                    raise
+                except Exception as exc:
+                    self.logger.error(f"{exc}")
+                    raise
+        else:
+            try:
+                await self.task.start()
+            except Exception as exc:
+                self.logger.error(exc)
+                raise TaskFailed(f"{exc!s}") from exc
+            try:
+                return await self.task.run()
+            except Exception as err:
+                raise TaskFailed(
+                    f"Error: Task {self.program}.{self.task} failed: {err}"
+                ) from err
+            finally:
+                await self.task.close()
+
+    async def close(self):
+        pass
+
+    async def open(self):
+        try:
+            if self._no_worker is False:
+                self.wrapper = TaskWrapper(
+                    program=self.program,
+                    task=self.task,
+                    **self._kwargs
+                )
+            else:
+                self.task = Task(
+                    task=self.task,
+                    program=self.program,
+                    debug=True
+                )
+        except Exception as exc:
+            self.logger.exception(str(exc), stack_info=True)
+            raise FlowTaskError(f"Error: {exc}") from exc

flowtask/interfaces/template.py

@@ -0,0 +1,118 @@
+from abc import ABC
+from pathlib import PurePath, Path
+import aiofiles
+from ..exceptions import FileError, FileNotFound
+from ..template import getTemplateHandler, TemplateHandler
+from ..conf import FILE_STORAGES, TASK_STORAGES, TASK_PATH
+
+
+class TemplateSupport(ABC):
+    """TemplateSupport.
+
+    Adding Support for Jinja2 Template parser on Components.
+    """
+    use_template: bool = False
+
+    def __init__(self, *args, **kwargs):
+        self._filestore = None
+        self._taskstorage = None
+        # Use Template:
+        self.use_template: bool = kwargs.pop(
+            'use_template', self.use_template
+        )
+        self.from_templates_dir: bool = kwargs.pop(
+            'from_templates_dir', False
+        )
+        # Template directory
+        template_dir = kwargs.pop('template_dir', None)
+        try:
+            super().__init__(*args, *args)
+        except TypeError:
+            super().__init__()
+        # Template Parser:
+        self._templateparser: TemplateHandler = None
+        if self.use_template is True:
+            self._templateparser = getTemplateHandler(
+                newdir=template_dir
+            )
+
+    async def open_templatefile(
+        self,
+        file: PurePath,
+        program: str = None,
+        from_templates_dir: bool = False,
+        folder: str = 'templates',
+        **kwargs
+    ) -> str:
+        """
+        Open a file, replace masks and parse template if needed.
+        """
+        if self._filestore is None:
+            # we need to calculate which is the filestore
+            self._filestore = FILE_STORAGES.get('default')
+        if self._taskstorage is None:
+            self._taskstorage = TASK_STORAGES.get('default')
+        if isinstance(file, str):
+            file = Path(file)
+        if not file.is_absolute():
+            # File is relative to TaskStorage:
+            if from_templates_dir is True:
+                directory = Path(TASK_PATH).parent.joinpath(folder)
+            else:
+                # Getting from default Task Storage:
+                directory = self._taskstorage.get_path().joinpath(program, folder)
+                if not directory.exists():
+                    # Get Template From Task Path:
+                    directory = Path(TASK_PATH).joinpath(program, folder)
+                    if not directory.exists():
+                        # Get Templates Dir from FileStore
+                        directory = self._filestore.get_directory(folder, program=program)
+            file = directory.joinpath(file).resolve()
+        if file.exists() and file.is_file():
+            content = None
+            # open File:
+            try:
+                async with aiofiles.open(file, "r+") as afp:
+                    content = await afp.read()
+            except Exception as exc:
+                raise FileError(
+                    f"{__name__}: Error Opening File: {file}"
+                ) from exc
+            if self.use_template is True:
+                content = self._templateparser.from_string(
+                    content,
+                    kwargs
+                )
+            elif hasattr(self, "masks"):
+                content = self.mask_replacement(content)
+            return content
+        else:
+            raise FileNotFound(
+                f"{__name__}: Missing File: {file}"
+            )
+
+    async def open_tmpfile(self, file: PurePath, **kwargs) -> str:
+        if file.exists() and file.is_file():
+            content = None
+            # open File:
+            try:
+                async with aiofiles.open(file, "r+") as afp:
+                    content = await afp.read()
+            except Exception as exc:
+                raise FileError(
+                    f"{self.__name__}: {exc}"
+                )
+                # check if we need to replace masks
+            if hasattr(self, "masks"):
+                self._logger.debug(
+                    f"TemplateSupport Masks: {self.masks}"
+                )
+                if "{" in content:
+                    content = self.mask_replacement(content)
+            if self.use_template is True:
+                content = self._templateparser.from_string(content, kwargs)
+            return content
+        else:
+            raise FileNotFound(
+                f"{self.__name__}: Missing File: {file}"
+            )

flowtask/interfaces/vectorstores/__init__.py

@@ -0,0 +1 @@
+from .milvus import MilvusStore

flowtask/interfaces/vectorstores/abstract.py

@@ -0,0 +1,133 @@
+from abc import ABC, abstractmethod
+import torch
+from langchain_huggingface import (
+    HuggingFaceEmbeddings
+)
+from langchain_community.embeddings import (
+    HuggingFaceBgeEmbeddings
+)
+from navconfig.logging import logging
+from ...conf import (
+    MAX_BATCH_SIZE,
+    EMBEDDING_DEVICE,
+    EMBEDDING_DEFAULT_MODEL
+)
+
+class AbstractStore(ABC):
+    """AbstractStore class.
+
+    Args:
+        embedding_model (dict): Embeddings.
+    """
+    def __init__(self, *args, **kwargs):
+        self.embedding_model: dict = kwargs.pop(
+            'embedding_model',
+            self._default_embedding_model()
+        )
+        self.logger = logging.getLogger(__name__)
+        self._metric_type: str = kwargs.pop("metric_type", self._default_metric())
+        self._index_type: str = kwargs.pop("index_type", self._default_index())
+        self._dimension: int = kwargs.pop('dimension', 768)
+        self.vector_field: str = kwargs.pop('vector_field', 'vector')
+        self.text_field: str = kwargs.pop('text_field', 'text')
+        self.database: str = kwargs.pop('database', 'default')
+        super().__init__(*args, **kwargs)
+        # Embedding Model
+        self._embed_ = None
+        self._connection = None
+
+    def _default_metric(self) -> str:
+        return 'COSINE'
+
+    def _default_index(self) -> str:
+        return 'IVF_FLAT'
+
+    def _default_embedding_model(self) -> dict:
+        return {
+            "model_name": "sentence-transformers/paraphrase-MiniLM-L6-v2",
+            "model_type": "transformers"
+        }
+
+    def _get_device(self, device_type: str = None, cuda_number: int = 0):
+        """Get Default device for Torch and transformers.
+
+        """
+        torch.backends.cudnn.deterministic = True
+        torch.backends.cudnn.benchmark = False
+        if device_type:
+            return torch.device(device_type)
+        if device_type == 'cpu' or EMBEDDING_DEVICE == 'cpu':
+            return torch.device('cpu')
+        if torch.cuda.is_available():
+            # Use CUDA GPU if available
+            return torch.device(f'cuda:{cuda_number}')
+        if torch.backends.mps.is_available():
+            # Use CUDA Multi-Processing Service if available
+            return torch.device("mps")
+        if EMBEDDING_DEVICE == 'cuda':
+            return torch.device(f'cuda:{cuda_number}')
+        else:
+            return torch.device(EMBEDDING_DEVICE)
+
+    def create_embedding(
+        self,
+        embedding_model: dict
+    ):
+        encode_kwargs: str = {
+            'normalize_embeddings': True,
+            "batch_size": MAX_BATCH_SIZE
+        }
+        device = self._get_device()
+        model_kwargs: str = {'device': device}
+        model_name = embedding_model.get('model_name', EMBEDDING_DEFAULT_MODEL)
+        model_type = embedding_model.get('model_type', 'transformers')
+        if model_type == 'bge':
+            return HuggingFaceBgeEmbeddings(
+                model_name=model_name,
+                model_kwargs=model_kwargs,
+                encode_kwargs=encode_kwargs
+            )
+        else:
+            return HuggingFaceEmbeddings(
+                model_name=model_name,
+                model_kwargs=model_kwargs,
+                encode_kwargs=encode_kwargs
+            )
+
+    def get_default_embedding(
+        self,
+        model_name: str = EMBEDDING_DEFAULT_MODEL
+    ):
+        return self.create_embedding(model_name=model_name)
+
+    @abstractmethod
+    async def load_documents(
+        self,
+        documents: list,
+        collection: str = None
+    ):
+        pass
+
+    @abstractmethod
+    def connect(self) -> None:
+        pass
+
+    @abstractmethod
+    def disconnect(self) -> None:
+        pass
+
+    async def __aenter__(self):
+        if self._embed_ is None:
+            self._embed_ = self.create_embedding(
+                self.embedding_model
+            )
+        await self.connect()
+        return self
+
+    async def __aexit__(self, exc_type, exc_value, traceback):
+        # closing Embedding
+        self._embed_ = None
+        try:
+            await self.disconnect()
+        except RuntimeError:
+            pass