blackant-sdk 1.0.2__py3-none-any.whl

storage/errors.py

@@ -0,0 +1,10 @@
+class InvalidStorageUsageException(Exception):
+    pass
+
+
+class StorageOperationFailedException(Exception):
+    pass
+
+
+class StorageCreationFailedException(Exception):
+    pass

storage/factory.py

@@ -0,0 +1,26 @@
+import os
+
+import urllib3
+
+from storage.errors import StorageCreationFailedException
+from storage.minio import MinioStorageManager
+
+
+class StorageFactory:  # pylint: disable=too-few-public-methods
+    __instance = None
+
+    @classmethod
+    def create(cls, logger=None):
+        if cls.__instance is None:
+            try:
+                cls.__instance = MinioStorageManager(
+                    os.environ["OBJECT_STORAGE_URL"],
+                    os.environ["CALCULATION_NAME"],
+                    os.environ["OBJECT_STORAGE_ACCESS_KEY"],
+                    os.environ["OBJECT_STORAGE_SECRET_KEY"],
+                    bool(int(os.environ["OBJECT_STORAGE_USE_SSL"])),
+                    logger=logger,
+                )
+            except urllib3.exceptions.MaxRetryError as exc:
+                raise StorageCreationFailedException from exc
+        return cls.__instance

storage/interface.py

@@ -0,0 +1,19 @@
+import abc
+
+
+class StorageInterface(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def download_file(self, local_path, file_id):
+        pass
+
+    @abc.abstractmethod
+    def upload_file(self, local_path, file_id):
+        pass
+
+    @abc.abstractmethod
+    def download_string(self, file_id):
+        pass
+
+    @abc.abstractmethod
+    def upload_string(self, input_string, file_id):
+        pass

storage/minio.py

@@ -0,0 +1,106 @@
+import io
+from datetime import timedelta
+import urllib3
+import minio
+
+from blackant.utils.logging import get_logger
+from storage.interface import StorageInterface
+
+
+class MinioStorageManager(StorageInterface):
+    __instance = None
+
+    def __init__(self, url, bucket_id, access_key, secret_key, secure, logger=None):  # pylint: disable=too-many-arguments
+        self.__bucket_id = "".join(letter for letter in bucket_id if letter.isalnum()).lower()
+        self.__logger = logger or get_logger("minio_client")
+
+        self.__logger.debug("Creating minio client for %s with user %s", url, access_key)
+        if not secure:
+            self.__logger.warning(
+                "Minio client is not using SSL. "
+                "If the client does NOT communicate via Docker inside network, "
+                "it should NOT run in production."
+            )
+
+        timeout = timedelta(minutes=5).seconds
+        http_client_pool_manager = urllib3.PoolManager(
+            timeout=urllib3.util.Timeout(connect=timeout, read=timeout),
+            maxsize=10,
+            cert_reqs="CERT_NONE",
+            retries=urllib3.Retry(
+                total=5, backoff_factor=0.2, status_forcelist=[500, 502, 503, 504]
+            ),
+        )
+
+        self.__client = minio.Minio(
+            url,
+            access_key=access_key,
+            secret_key=secret_key,
+            secure=secure,
+            http_client=http_client_pool_manager,
+        )
+
+        try:
+            if not self.__client.bucket_exists(self.__bucket_id):
+                self.__client.make_bucket(self.__bucket_id)
+        except minio.error.AccessDenied:
+            self.__logger.critical(
+                "Cannot access object storage. Access is denied. "
+                "Object storage url: %s , "
+                "Access key: %s",
+                url,
+                access_key,
+            )
+
+    def download_file(self, local_path, file_id):
+        try:
+            self.__client.fget_object(self.__bucket_id, file_id, local_path)
+        except minio.error.AccessDenied:
+            self.__logger.critical(
+                "Cannot download file from object storage. Access is denied. "
+                "Bucket: %s, remote: %s, local: %s",
+                self.__bucket_id,
+                file_id,
+                local_path,
+            )
+
+    def upload_file(self, local_path, file_id):
+        try:
+            self.__client.fput_object(self.__bucket_id, file_id, local_path)
+        except minio.error.AccessDenied:
+            self.__logger.critical(
+                "Cannot upload file to object storage. Access is denied. "
+                "Bucket: %s, remote: %s, local: %s",
+                self.__bucket_id,
+                file_id,
+                local_path,
+            )
+
+    def download_string(self, file_id):
+        try:
+            obj = self.__client.get_object(self.__bucket_id, file_id)
+            return obj.data.decode("utf-8")
+        except minio.error.AccessDenied:
+            self.__logger.critical(
+                "Cannot download file as string from object storage. Access is denied. "
+                "Bucket: %s, remote: %s",
+                self.__bucket_id,
+                file_id,
+            )
+        return ""
+
+    def upload_string(self, input_string, file_id):
+        try:
+            value_as_bytes = input_string.encode("utf-8")
+            value_as_a_stream = io.BytesIO(value_as_bytes)
+            self.__client.put_object(
+                self.__bucket_id, file_id, value_as_a_stream, length=len(value_as_bytes)
+            )
+        except minio.error.AccessDenied:
+            self.__logger.critical(
+                "Cannot upload string to file to object storage. Access is denied. "
+                "Bucket: %s, remote: %s, string: %s",
+                self.__bucket_id,
+                file_id,
+                input_string,
+            )

task/dao.py

@@ -0,0 +1,38 @@
+import uuid
+
+from task.errors import InvalidDAOUsageError, TaskDoesNotExistsError
+
+
+class TaskDao:
+    __instance = None
+
+    def __init__(self):
+        if self.__instance is not None:
+            raise InvalidDAOUsageError("Singleton object was created multiple times")
+        self.__data = {}
+
+    @classmethod
+    def get_instance(cls):
+        if cls.__instance is None:
+            cls.__instance = TaskDao()
+        return cls.__instance
+
+    def get_task_ids(self):
+        return list(self.__data.keys())
+
+    def get_task(self, task_id):
+        if task_id not in self.__data.keys():
+            raise TaskDoesNotExistsError
+        return self.__data.get(task_id)
+
+    def add_task(self, task):
+        task_id = uuid.uuid4().hex
+        task.task_id = task_id
+        self.__data[task_id] = task
+        return task_id
+
+    def delete_task(self, task_id):
+        if task_id not in self.__data.keys():
+            raise TaskDoesNotExistsError
+        self.__data.pop(task_id)
+        return task_id

task/errors.py

@@ -0,0 +1,10 @@
+class InvalidDAOUsageError(Exception):
+    pass
+
+
+class TaskDoesNotExistsError(Exception):
+    pass
+
+
+class TaskCallbackFailedError(Exception):
+    pass

task/log_adapter.py

@@ -0,0 +1,11 @@
+import logging
+
+
+class TaskIdAdapter(logging.LoggerAdapter):
+    def set_task_id(self, task_id):
+        self.extra["task_id"] = task_id
+
+    def process(self, msg, kwargs):
+        if "task_id" in self.extra:
+            return "[%s] %s" % (self.extra["task_id"], msg), kwargs
+        return "%s" % msg, kwargs

task/parsers/base.py

@@ -0,0 +1,13 @@
+import abc
+
+from flask_restful import reqparse
+
+
+class RequestParserBase(reqparse.RequestParser):
+    @abc.abstractmethod
+    def validate(self, request):
+        pass
+
+    @abc.abstractmethod
+    def parse(self, request):
+        pass

task/parsers/callback.py

@@ -0,0 +1,40 @@
+from dataclasses import dataclass
+
+from task.parsers.base import RequestParserBase
+
+
+@dataclass
+class CallbackParameter:
+    url: str
+    method: str
+    headers: dict
+    request_body: dict
+
+
+class CallbackParser(RequestParserBase):
+    def __init__(self, loc):
+        super().__init__()
+
+        self.add_argument("url", type=str, location=loc)
+        self.add_argument(
+            "method",
+            type=str,
+            location=loc,
+            default="GET",
+            choices=("GET", "POST", "PUT", "DELETE"),
+        )
+        self.add_argument("headers", type=dict, location=loc, default={})
+        self.add_argument("request_body", type=dict, location=loc, default={})
+
+    def validate(self, request):
+        self.parse_args(req=request)
+
+    def parse(self, request):
+        result = self.parse_args(req=request)
+
+        return CallbackParameter(
+            url=result["url"],
+            method=result["method"],
+            headers=result["headers"],
+            request_body=result["request_body"],
+        )

task/parsers/cmd_args.py

@@ -0,0 +1,52 @@
+import sys
+import ast
+
+import werkzeug.exceptions
+
+from blackant.utils.logging import get_logger
+from task.parsers.base import RequestParserBase
+
+
+class CommandLineArgumentParser(RequestParserBase):
+    def __init__(self, loc):
+        super().__init__()
+
+        self.__loc = loc
+        self.__logger = get_logger("cmd_args")
+
+    @staticmethod
+    def check_size_of_data(request_field, cmd_data):
+        size_of_data = sys.getsizeof(cmd_data)
+        if size_of_data > 35000000:
+            raise werkzeug.exceptions.BadRequest(
+                "The getting '{}' parameter size is too big. "
+                "The current data size is {} bytes. "
+                "35000000 bytes allows".format(request_field, size_of_data)
+            )
+
+    def validate(self, request):
+        if not isinstance(request[self.__loc], dict):
+            raise werkzeug.exceptions.BadRequest("Command line argument input is not a dictionary")
+
+        self.check_size_of_data(self.__loc, request)
+
+        for key, value in request[self.__loc].items():
+            if not isinstance(key, str) or not isinstance(value, str):
+                raise werkzeug.exceptions.BadRequest(
+                    "Command line argument input has invalid key or value"
+                )
+
+    def parse(self, request):
+        return_data = {}
+
+        if not request or self.__loc not in request.keys():
+            return return_data
+
+        self.check_size_of_data(self.__loc, request)
+
+        request_data = request[self.__loc]
+
+        if isinstance(request_data, str):
+            request_data = ast.literal_eval(request_data)
+
+        return request_data

task/parsers/freetext.py

@@ -0,0 +1,19 @@
+import werkzeug.exceptions
+
+from task.parsers.base import RequestParserBase
+
+
+class FreeTextInputParser(RequestParserBase):
+    def __init__(self, loc):
+        super().__init__()
+
+        self.__loc = loc
+
+    def validate(self, request):
+        if not isinstance(request[self.__loc], str):
+            raise werkzeug.exceptions.BadRequest("Freetext input must be text type")
+
+    def parse(self, request):
+        if not request or self.__loc not in request.keys():
+            return ""
+        return request[self.__loc]

task/parsers/objects.py

@@ -0,0 +1,50 @@
+from dataclasses import dataclass
+
+import werkzeug.exceptions
+
+from task.parsers.base import RequestParserBase
+
+
+@dataclass
+class ObjectParameter:
+    remote: str
+    local: str
+    preserve: bool = False
+
+
+class ObjectListParser(RequestParserBase):
+    def __init__(self, loc):
+        super().__init__()
+
+        self.__loc = loc
+
+    def validate(self, request):
+        if not isinstance(request[self.__loc], list):
+            raise werkzeug.exceptions.BadRequest("Objects input is not a list")
+
+        for obj in request[self.__loc].items():
+            if not isinstance(obj, dict):
+                raise werkzeug.exceptions.BadRequest("Objects input has invalid value")
+
+            if "remote" not in obj.keys() or "local" not in obj.keys():
+                raise werkzeug.exceptions.BadRequest("Object input has missing mandatory element")
+
+            if not isinstance(obj["remote"], str) or not isinstance(obj["local"], str):
+                raise werkzeug.exceptions.BadRequest("Object input has invalid mandatory element")
+
+    def parse(self, request):
+        try:
+            request = request[self.__loc]
+            if not request:
+                return []
+            return [
+                ObjectParameter(
+                    remote=request["remote"],
+                    local=request["local"],
+                    preserve=request["preserve"] if "preserve" in request.keys() else False,
+                )
+                for obj in request[self.__loc]
+            ]
+        except KeyError:
+            pass
+        return []

task/parsers/request.py

@@ -0,0 +1,56 @@
+import dataclasses
+import typing
+from task.parsers.base import RequestParserBase
+
+from task.parsers.callback import CallbackParser, CallbackParameter
+from task.parsers.cmd_args import CommandLineArgumentParser
+from task.parsers.objects import ObjectListParser, ObjectParameter
+from task.parsers.freetext import FreeTextInputParser
+
+
+@dataclasses.dataclass
+class TaskConfiguration:
+    callback: CallbackParameter
+    cmd_args: typing.List[typing.Dict]
+    objects: typing.List[ObjectParameter]
+    input: str
+
+    @property
+    def json(self):
+        return {
+            "callback": dataclasses.asdict(self.callback),
+            "cmd_args": self.cmd_args,
+            "objects": [dataclasses.asdict(obj) for obj in self.objects],
+            "input": self.input,
+        }
+
+
+class RequestParser(RequestParserBase):
+    def __init__(self):
+        super().__init__()
+
+        self.add_argument("callback", type=dict)
+        self.add_argument("input")
+        self.add_argument("objects")
+        self.add_argument("cmd_args")
+
+        self.__callback_parser = CallbackParser("callback")
+        self.__cmd_args_parser = CommandLineArgumentParser("cmd_args")
+        self.__objects_parser = ObjectListParser("objects")
+        self.__freetext_parser = FreeTextInputParser("input")
+
+    def validate(self, request):
+        self.__callback_parser.validate(request)
+        self.__cmd_args_parser.validate(request)
+        self.__objects_parser.validate(request)
+        self.__freetext_parser.validate(request)
+
+    def parse(self, request):
+        request = self.parse_args()
+
+        return TaskConfiguration(
+            callback=self.__callback_parser.parse(request),
+            cmd_args=self.__cmd_args_parser.parse(request),
+            objects=self.__objects_parser.parse(request),
+            input=self.__freetext_parser.parse(request),
+        )

task/resource.py

@@ -0,0 +1,84 @@
+import logging
+import os
+import tempfile
+import uuid
+
+from flask import jsonify, Response
+from flask_restful import Resource
+
+from blackant.utils.logging import get_logger
+from task.task import Task
+from task.dao import TaskDao
+from task.errors import TaskDoesNotExistsError
+from task.log_adapter import TaskIdAdapter
+from task.parsers.request import RequestParser
+
+
+class TaskResource(Resource):
+    def __init__(self):
+        self.__task_dao = TaskDao.get_instance()
+
+    def get(self, task_id=None):
+        if task_id is not None:
+            try:
+                task = self.__task_dao.get_task(task_id)
+                response = jsonify(task.response)
+            except TaskDoesNotExistsError:
+                response = Response(status=404)
+        else:
+            response = jsonify(self.__task_dao.get_task_ids())
+        return response
+
+    def post(self):
+        configuration = RequestParser().parse(None)
+        unique_uuid = uuid.uuid4().hex
+        tmp_dir = tempfile.gettempdir()
+
+        log_file_handler = logging.FileHandler(
+            os.path.join(tmp_dir, "user_log_{}.log".format(unique_uuid)), mode="w"
+        )
+        log_file_handler.setLevel(logging.DEBUG)
+
+        calculation_logger_instance = get_logger("calculation_{}".format(unique_uuid))
+        calculation_logger_instance.setLevel(logging.DEBUG)
+        calculation_logger_instance.addHandler(log_file_handler)
+        calculation_logger = TaskIdAdapter(calculation_logger_instance, {})
+
+        task_logger_instance = get_logger("task_{}".format(unique_uuid))
+        task_logger_instance.setLevel(logging.DEBUG)
+        task_logger_instance.addHandler(log_file_handler)
+        task_logger = TaskIdAdapter(task_logger_instance, {})
+
+        cmd_args = configuration.cmd_args if configuration.cmd_args else {}
+        try:
+            # Dynamic calculation loading using CalculationLoader
+            # Supports environment variable configuration (CALCULATION_MODULE, CALCULATION_CLASS)
+            from calculation.loader import CalculationLoader  # pylint: disable=import-outside-toplevel
+
+            loader = CalculationLoader()
+            CalculationClass = loader.get_calculation()
+            calculation = CalculationClass(calculation_logger, **cmd_args)
+        # The too-broad exception warning is suppressed because we want to handle exceptions
+        except Exception:  # pylint: disable=broad-except
+            task_logger.exception("Cannot create the calculation instance.")
+            calculation = None
+        new_task = Task(calculation, configuration, log_file_handler, task_logger)
+
+        new_task_id = self.__task_dao.add_task(new_task)
+        task_logger.info("The calculated task UUID: {}".format(new_task_id))
+        calculation_logger.set_task_id(new_task_id)
+        task_logger.set_task_id(new_task_id)
+
+        new_task.run()
+
+        response = jsonify(new_task_id)
+        return response
+
+    def delete(self, task_id):
+        try:
+            task = self.__task_dao.get_task(task_id)
+            task.stop()
+            response = jsonify(self.__task_dao.delete_task(task_id))
+        except TaskDoesNotExistsError:
+            response = Response(status=404)
+        return response

task/states/base.py

@@ -0,0 +1,14 @@
+import abc
+
+
+class TaskStateBase:
+    def __init__(self, task):
+        self._task = task
+
+    @abc.abstractproperty
+    def name(self):
+        raise NotImplementedError
+
+    @abc.abstractmethod
+    def run(self):
+        raise NotImplementedError

task/states/error.py

@@ -0,0 +1,47 @@
+from pathlib import Path
+from threading import Lock
+
+from task.states.base import TaskStateBase
+from task.errors import TaskCallbackFailedError
+
+log_lock = Lock()
+
+
+class TaskErrorState(TaskStateBase):
+    @property
+    def name(self):
+        return "error"
+
+    def run(self):
+        self._task.logger.info("Calculation went to error state")
+
+        # Get the log file path and read its content
+        log_file_path = Path(self._task.log_file_handler.baseFilename)
+        log_content = log_file_path.read_text()
+
+        # Truncate the log if it exceeds the maximum allowed size (20,000 characters)
+        if len(log_content) > 20000:
+            truncated_log = f"{log_content[:10000]}\n...\n...\n...\n{log_content[-10000:]}"
+        else:
+            truncated_log = log_content
+
+        # Attempt to send callbacks with the error and log information
+        try:
+            self._task.send_callbacks(
+                {"error": "Calculation went to error state", "log": truncated_log}
+            )
+        except TaskCallbackFailedError as exception:
+            self._task.logger.critical("Cannot send callback")
+            self._task.logger.exception(exception)
+
+        self._task.set_finished(True)
+
+        # Safely remove log handler and delete the log file
+        with log_lock:
+            try:
+                # Remove the log file handler from the logger
+                self._task.logger.logger.handlers.remove(self._task.log_file_handler)
+                if log_file_path.exists():
+                    log_file_path.unlink()  # Delete the log file
+            except (OSError, ValueError) as exc:
+                self._task.logger.warning(f"Cannot safely remove or delete the log file: {exc}")

task/states/idle.py

@@ -0,0 +1,12 @@
+from task.states.base import TaskStateBase
+from task.states.set_up import TaskSetUpState
+
+
+class TaskIdleState(TaskStateBase):
+    @property
+    def name(self):
+        return "idle"
+
+    def run(self):
+        self._task.logger.info("Start processing")
+        self._task.change_state(TaskSetUpState(self._task))

task/states/ready.py

@@ -0,0 +1,51 @@
+from pathlib import Path
+from threading import Lock
+
+from task.errors import TaskCallbackFailedError
+from task.states.base import TaskStateBase
+from task.states.error import TaskErrorState
+
+log_lock = Lock()
+
+
+class TaskReadyState(TaskStateBase):
+    @property
+    def name(self):
+        return "ready"
+
+    def run(self):
+        self._task.logger.info("Calculation has finished")
+
+        # Read and truncate the log file if necessary
+        log_file_path = Path(self._task.log_file_handler.baseFilename)
+        calculation_log = log_file_path.read_text()
+
+        if len(calculation_log) > 20000:
+            calculation_log = (
+                f"{calculation_log[:10000]}\n...\n...\n...\n{calculation_log[-10000:]}"
+            )
+
+        try:
+            # Send callback with result and truncated log
+            self._task.send_callbacks(
+                {"result": self._task.calculation.result, "log": calculation_log}
+            )
+            self._task.set_finished(True)
+        except TaskCallbackFailedError as ex:
+            self._task.logger.critical("Cannot send callback. Transitioning to error state.")
+            self._task.logger.exception(ex)
+            self._task.change_state(TaskErrorState(self._task))
+            # Early return to avoid cleaning up the log file; error state may need it.
+            return
+
+        # Safely remove log handler and delete log file
+        with log_lock:
+            try:
+                self._task.logger.logger.handlers.remove(self._task.log_file_handler)
+                if log_file_path.exists():
+                    log_file_path.unlink()  # Delete log file
+                    self._task.logger.info(f"Log file {log_file_path} successfully removed.")
+            except (OSError, ValueError) as exc:
+                self._task.logger.warning(
+                    f"Failed to remove or delete log file ({log_file_path}): {exc}"
+                )