ul-api-utils 9.3.0__py3-none-any.whl

ul_api_utils/utils/api_format.py

@@ -0,0 +1,61 @@
+import json
+from enum import Enum
+from typing import Tuple, Any, Optional
+
+import msgpack
+
+from ul_api_utils.const import MIME__JSON, MIME__MSGPCK
+from ul_api_utils.utils.json_encoder import CustomJSONEncoder
+
+
+class ApiFormat(Enum):
+    JSON = 'JSON'
+    MESSAGE_PACK = 'MESSAGE_PACK'
+
+    def __repr__(self) -> str:
+        return f'{type(self).__name__}.{self.name}'
+
+    @staticmethod
+    def accept_mimes(force: Optional['ApiFormat'] = None) -> Tuple[str, ...]:
+        if force is ApiFormat.MESSAGE_PACK:
+            return MIME__MSGPCK,  # noqa: C818
+        if force is ApiFormat.JSON:
+            return MIME__JSON,  # noqa: C818
+        return MIME__MSGPCK, MIME__JSON
+
+    @property
+    def mime(self) -> str:
+        if self is ApiFormat.MESSAGE_PACK:
+            return MIME__MSGPCK
+        if self is ApiFormat.JSON:
+            return MIME__JSON
+        raise NotImplementedError
+
+    @staticmethod
+    def from_mime(mime: str) -> Optional['ApiFormat']:
+        if mime == MIME__MSGPCK:
+            return ApiFormat.MESSAGE_PACK
+        if mime == MIME__JSON or (mime.startswith("application/") and mime.endswith("+json")):
+            return ApiFormat.JSON
+        return None
+
+    def serialize_bytes(self, data: Any) -> bytes:
+        if self is ApiFormat.MESSAGE_PACK:
+            return msgpack.dumps(data, default=CustomJSONEncoder().default)
+        if self is ApiFormat.JSON:
+            return json.dumps(data, cls=CustomJSONEncoder).encode('utf-8')
+        raise NotImplementedError
+
+    def parse_bytes(self, data: bytes) -> Any:
+        if self is ApiFormat.MESSAGE_PACK:
+            return msgpack.unpackb(data, use_list=True)
+        if self is ApiFormat.JSON:
+            return json.loads(data)
+        raise NotImplementedError
+
+    def parse_text(self, data: str) -> Any:
+        if self is ApiFormat.MESSAGE_PACK:
+            return msgpack.unpackb(data.encode('utf-8'), use_list=True)
+        if self is ApiFormat.JSON:
+            return json.loads(data)
+        raise NotImplementedError

ul_api_utils/utils/api_method.py

@@ -0,0 +1,55 @@
+from enum import Enum, unique
+from typing import Union, List, Tuple, Any, Literal
+
+from ul_api_utils.const import REQUEST_METHOD__PUT, REQUEST_METHOD__GET, REQUEST_METHOD__POST, \
+    REQUEST_METHOD__PATCH, REQUEST_METHOD__DELETE, REQUEST_METHOD__OPTIONS, REQUEST_METHOD__QUERY
+
+TMethodStr = Union[Literal['GET'], Literal['POST'], Literal['PUT'], Literal['PATCH'], Literal['DELETE'], Literal['OPTIONS']]
+TMethod = Union[TMethodStr, 'ApiMethod', List[TMethodStr], Tuple[TMethodStr, ...], List['ApiMethod'], Tuple['ApiMethod', ...]]
+TMethodShort = Union[TMethodStr, 'ApiMethod']
+
+
+@unique
+class ApiMethod(Enum):
+    PUT = REQUEST_METHOD__PUT
+    GET = REQUEST_METHOD__GET
+    POST = REQUEST_METHOD__POST
+    QUERY = REQUEST_METHOD__QUERY
+    PATCH = REQUEST_METHOD__PATCH
+    DELETE = REQUEST_METHOD__DELETE
+    OPTIONS = REQUEST_METHOD__OPTIONS
+
+    def __repr__(self) -> str:
+        return f'{type(self).__name__}.{self.name}'
+
+    def __hash__(self) -> int:
+        return hash(self.value)
+
+    def __str__(self) -> str:
+        return self.value
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, str):
+            try:
+                other = type(self)(other.upper())
+            except Exception:  # noqa: B902
+                return False
+        return super(ApiMethod, self).__eq__(other)
+
+    @staticmethod
+    def compile_methods(methods: TMethod) -> Tuple[List[str], List['ApiMethod']]:
+        str_res_methods = []
+        enum_res_methods = []
+        for m in (list(methods) if isinstance(methods, (list, tuple)) else [methods]):
+            if isinstance(m, ApiMethod):
+                str_res_methods.append(m.value)
+                enum_res_methods.append(m)
+            else:
+                assert isinstance(m, str)
+                m = m.strip().upper()  # type: ignore
+                enum_res_methods.append(ApiMethod(m))
+                str_res_methods.append(m)
+        return str_res_methods, enum_res_methods
+
+
+NO_REQUEST_BODY_METHODS = {ApiMethod.GET, ApiMethod.OPTIONS}

ul_api_utils/utils/api_pagination.py

@@ -0,0 +1,58 @@
+from typing import Iterable, Any, Optional, Iterator
+
+import math
+
+
+class ApiPagination:
+    def __init__(self, page: int, per_page: int, total: int, items: Iterable[Any]) -> None:
+        self.query = None
+        self.page = page
+        self.per_page = per_page
+        self.total = total
+        self.items = items
+
+    @property
+    def pages(self) -> int:
+        """The total number of pages"""
+        if self.per_page == 0:
+            pages = 0
+        else:
+            pages = int(math.ceil(self.total / float(self.per_page)))
+        return pages
+
+    @property
+    def prev_num(self) -> Optional[int]:
+        """Number of the previous page."""
+        if not self.has_prev:
+            return None
+        return self.page - 1
+
+    @property
+    def has_prev(self) -> bool:
+        """True if a previous page exists"""
+        return self.page > 1
+
+    def prev(self, error_out: bool = False) -> None:
+        raise AssertionError('a query object is required for this method to work')
+
+    def next(self, error_out: bool = False) -> None:
+        raise AssertionError('a query object is required for this method to work')
+
+    @property
+    def has_next(self) -> bool:
+        return self.page < self.pages
+
+    @property
+    def next_num(self) -> Optional[int]:
+        if not self.has_next:
+            return None
+        return self.page + 1
+
+    def iter_pages(self, left_edge: int = 2, left_current: int = 2, right_current: int = 5, right_edge: int = 2) -> Iterator[Optional[int]]:
+        last = 0
+        for num in range(1, self.pages + 1):
+            if num <= left_edge or (num > self.page - left_current - 1 and num < self.page + right_current) or num > self.pages - right_edge:
+                if last + 1 != num:
+                    yield None
+                yield num
+                last = num

ul_api_utils/utils/api_path_version.py

@@ -0,0 +1,60 @@
+from enum import Enum
+from typing import Optional, Mapping, Union, Dict, Any
+
+
+class ApiPathVersion(Enum):
+    V01 = 'v1'
+    V02 = 'v2'
+    V03 = 'v3'
+    V04 = 'v4'
+    V05 = 'v5'
+    V06 = 'v6'
+    V07 = 'v7'
+    V08 = 'v8'
+    V09 = 'v9'
+    V10 = 'v10'
+
+    NO_VERSION = "no-version"
+    NO_PREFIX = 'no-prefix'
+
+    def __repr__(self) -> str:
+        return f'{type(self).__name__}.{self.name}'
+
+    @staticmethod
+    def cleanup_q(q: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+        if q is None:
+            return None
+        res = {}
+        for k, v in q.items():
+            if v is not None:
+                res[k] = v
+        return res
+
+    def compile_path(self, path: str, prefix: str = "", *, q: Optional[Mapping[str, Union[int, str]]] = None) -> str:
+        assert '?' not in path, f'restricted symbol "?" was found in url.path="{path}"'
+        assert '&' not in path, f'restricted symbol "&" was found in url.path="{path}"'
+        assert '?' not in prefix, f'restricted symbol "?" was found in url.prefix="{prefix}"'
+        assert '&' not in prefix, f'restricted symbol "&" was found in url.prefix="{prefix}"'
+
+        qr = ''
+        if q:
+            for k, kv in q.items():
+                if kv is None:
+                    continue  # type: ignore
+                qr += ('?' if len(qr) == 0 else '&') + f'{k}={kv}'
+
+        if self is ApiPathVersion.NO_PREFIX:
+            if not path:
+                return qr
+            return f'/{path.lstrip("/")}{qr}'
+
+        if self is ApiPathVersion.NO_VERSION:
+            if not path:
+                return f'{"/" if prefix else ""}{prefix.lstrip("/")}{qr}'
+            return f'{"/" if prefix else ""}{prefix.strip("/")}/{path.lstrip("/")}{qr}'
+
+        assert prefix, f'prefix must not be empty. {prefix} was given'
+
+        if not path:
+            return f'/{prefix.strip("/")}/{self.value}{qr}'
+        return f'/{prefix.strip("/")}/{self.value}/{path.lstrip("/")}{qr}'

ul_api_utils/utils/api_request_info.py

@@ -0,0 +1,6 @@
+from typing import NamedTuple
+
+
+class ApiRequestInfo(NamedTuple):
+    user_agent: str
+    ipv4: str

ul_api_utils/utils/avro.py

@@ -0,0 +1,131 @@
+from typing import Type, Dict, Any, Optional, List, Set
+
+from pydantic import BaseModel
+
+# CODE FROM https://github.com/godatadriven/pydantic-avro/blob/main/src/pydantic_avro/base.py
+
+
+def get_avro_schema(model: Type[BaseModel], by_alias: bool = True, namespace: Optional[str] = None) -> Dict[str, Any]:
+    schema = model.schema(by_alias=by_alias)
+    return {
+        "type": "record",
+        "namespace": schema["title"] if namespace is None else namespace,
+        "name": schema["title"],
+        "fields": _get_fields(schema, set()),
+    }
+
+
+def _get_type(schema: Dict[str, Any], value: Dict[str, Any], classes_seen: Set[str]) -> Dict[str, Any]:
+    """Returns a type of single field"""
+    t = value.get("type")
+    f = value.get("format")
+    r = value.get("$ref")
+    a = value.get("additionalProperties")
+    avro_type_dict: Dict[str, Any] = {}
+    if "default" in value:
+        avro_type_dict["default"] = value.get("default")
+    if "description" in value:
+        avro_type_dict["doc"] = value.get("description")
+    if "allOf" in value and len(value["allOf"]) == 1:
+        r = value["allOf"][0]["$ref"]
+    if r is not None:
+        class_name = r.replace("#/definitions/", "")
+        if class_name in classes_seen:
+            avro_type_dict["type"] = class_name
+        else:
+            d = _get_definition(r, schema)
+            if "enum" in d:
+                avro_type_dict["type"] = {
+                    "type": "enum",
+                    "symbols": [str(v) for v in d["enum"]],
+                    "name": d["title"],
+                }
+            else:
+                avro_type_dict["type"] = {
+                    "type": "record",
+                    "fields": _get_fields(d, classes_seen),
+                    # Name of the struct should be unique true the complete schema
+                    # Because of this the path in the schema is tracked and used as name for a nested struct/array
+                    "name": class_name,
+                }
+            classes_seen.add(class_name)
+    elif t == "array":
+        items: Dict[str, Any] = value.get("items")  # type: ignore
+        tn: Dict[str, Any] = _get_type(schema, items, classes_seen)
+        # If items in array are a object:
+        if "$ref" in items:
+            tn = tn["type"]
+        # If items in array are a logicalType
+        if (
+            isinstance(tn, dict)
+            and isinstance(tn.get("type", {}), dict)
+            and tn.get("type", {}).get("logicalType") is not None
+        ):
+            tn = tn["type"]
+        avro_type_dict["type"] = {"type": "array", "items": tn}
+    elif t == "string" and f == "date-time":
+        avro_type_dict["type"] = {
+            "type": "long",
+            "logicalType": "timestamp-micros",
+        }
+    elif t == "string" and f == "date":
+        avro_type_dict["type"] = {
+            "type": "int",
+            "logicalType": "date",
+        }
+    elif t == "string" and f == "time":
+        avro_type_dict["type"] = {
+            "type": "long",
+            "logicalType": "time-micros",
+        }
+    elif t == "string" and f == "uuid":
+        avro_type_dict["type"] = {
+            "type": "string",
+            "logicalType": "uuid",
+        }
+    elif t == "string":
+        avro_type_dict["type"] = "string"
+    elif t == "number":
+        avro_type_dict["type"] = "double"
+    elif t == "integer":
+        # integer in python can be a long
+        avro_type_dict["type"] = "long"
+    elif t == "boolean":
+        avro_type_dict["type"] = "boolean"
+    elif t == "object":
+        if a is None:
+            value_type = "string"
+        else:
+            value_type = _get_type(schema, a, classes_seen)  # type: ignore
+        if isinstance(value_type, dict) and len(value_type) == 1:  # type: ignore
+            value_type = value_type.get("type")  # type: ignore
+        avro_type_dict["type"] = {"type": "map", "values": value_type}
+    else:
+        raise NotImplementedError(f"Type '{t}' not support yet, please report this at https://github.com/godatadriven/pydantic-avro/issues")
+    return avro_type_dict
+
+
+def _get_definition(ref: str, schema: Dict[str, Any]) -> Dict[str, Any]:
+    """Reading definition of base schema for nested structs"""
+    id = ref.replace("#/definitions/", "")
+    d = schema.get("definitions", {}).get(id)
+    if d is None:
+        raise RuntimeError(f"Definition {id} does not exist")
+    return d
+
+
+def _get_fields(schema: Dict[str, Any], classes_seen: Set[str]) -> List[Dict[str, Any]]:
+    """Return a list of fields of a struct"""
+    fields = []
+    required = schema.get("required", [])
+    for key, value in schema.get("properties", {}).items():
+        avro_type_dict = _get_type(schema, value, classes_seen)
+        avro_type_dict["name"] = key
+
+        if key not in required:
+            if avro_type_dict.get("default") is None:
+                avro_type_dict["type"] = ["null", avro_type_dict["type"]]
+                avro_type_dict["default"] = None
+
+        fields.append(avro_type_dict)
+    return fields

ul_api_utils/utils/broker_topics_message_count.py

@@ -0,0 +1,47 @@
+from typing import NamedTuple, List, Optional
+
+from ul_unipipeline.errors import UniError
+from ul_unipipeline.modules.uni import Uni
+
+
+class DataStreamStats(NamedTuple):
+    messages_count: int
+    queue_name: str
+    error_queue: bool
+    error: Optional[str]
+
+
+def get_data_streams_stats(uni: Uni) -> List[DataStreamStats]:
+    stats = []
+    error__error_messages_count = error__expect_messages_count = None
+    for wd in uni.config.workers.values():
+        try:
+            broker = uni._mediator.get_broker(wd.broker.name)
+        except UniError:
+            pass
+        else:
+            try:
+                expect_messages_count = broker.get_topic_approximate_messages_count(wd.topic)
+            except UniError:
+                expect_messages_count = -1
+                error__expect_messages_count = "inactive"
+            try:
+                error_messages_count = broker.get_topic_approximate_messages_count(wd.error_topic)
+            except UniError:
+                error_messages_count = -1
+                error__error_messages_count = "inactive"
+
+            stats.append(DataStreamStats(
+                messages_count=expect_messages_count,
+                queue_name=wd.topic,
+                error_queue=False,
+                error=error__expect_messages_count,
+            ))
+            stats.append(DataStreamStats(
+                messages_count=error_messages_count,
+                queue_name=wd.error_topic,
+                error_queue=True,
+                error=error__error_messages_count,
+            ))
+
+    return stats

ul_api_utils/utils/cached_per_request.py

@@ -0,0 +1,23 @@
+import functools
+from typing import Callable, TypeVar, Any
+
+from flask import g
+
+TFn = TypeVar("TFn", bound=Callable[..., Any])
+
+
+DEFAULT_OBJ = object()
+
+
+def cached_per_request(key: str) -> Callable[[TFn], TFn]:
+    def wrapper(fn: Callable[[...], Any]) -> Any:  # type: ignore
+        @functools.wraps(fn)
+        def wr(*args: Any, **kwargs: Any) -> Any:
+            cached_res = getattr(g, key, DEFAULT_OBJ)
+            if cached_res is not DEFAULT_OBJ:
+                return cached_res
+            res = fn(*args, **kwargs)
+            setattr(g, key, res)
+            return res
+        return wr
+    return wrapper

ul_api_utils/utils/colors.py

@@ -0,0 +1,31 @@
+from typing import Dict
+
+C_NC = ""
+C_FG_RED = ""
+C_FG_GREEN = ""
+C_FG_YELLOW = ""
+C_FG_GRAY = ""
+C_FG_BLUE = ""
+
+COLORS_MAP__TERMINAL: Dict[str, str] = {
+    C_NC: C_NC,
+    C_FG_RED: C_FG_RED,
+    C_FG_GREEN: C_FG_GREEN,
+    C_FG_YELLOW: C_FG_YELLOW,
+    C_FG_GRAY: C_FG_GRAY,
+    C_FG_BLUE: C_FG_BLUE,
+    "": "",
+}
+
+
+COLORS_MAP__HTML = {
+    C_NC: '</span>',
+    C_FG_GRAY: '<span style="color: gray;">',
+    C_FG_RED: '<span style="color: red;">',
+    C_FG_YELLOW: '<span style="color: yellow;">',
+    C_FG_GREEN: '<span style="color: green;">',
+    C_FG_BLUE: '<span style="color: blue;">',
+    "": "",
+}
+
+assert set(COLORS_MAP__TERMINAL.keys()) == set(COLORS_MAP__HTML.keys())

ul_api_utils/utils/constants.py

@@ -0,0 +1,7 @@
+from typing import Any, Dict, TypeVar, Tuple, Union
+
+from flask import Response
+
+TKwargs = TypeVar('TKwargs', bound=Dict[str, Any])
+TJsonResponse = Union[Tuple[str, int], Tuple[Response, int]]
+TJsonObj = Dict[str, Any]

ul_api_utils/utils/decode_base64.py

@@ -0,0 +1,9 @@
+import base64
+
+
+def decode_base64_to_string(encoded_str: str) -> str:
+    assert isinstance(encoded_str, str) and len(encoded_str) > 0, f'String with item required. Object {encoded_str} with type {type(encoded_str)} was given'
+    encoded_bytes = encoded_str.encode('utf-8')
+    decoded_bytes = base64.b64decode(encoded_bytes)
+    decoded_str = decoded_bytes.decode('utf-8')
+    return decoded_str

ul_api_utils/utils/deprecated.py

@@ -0,0 +1,19 @@
+import functools
+import warnings
+from typing import TypeVar, Callable, cast
+
+TFunc = TypeVar('TFunc', bound=Callable)  # type: ignore
+
+
+# https://stackoverflow.com/questions/2536307/decorators-in-the-python-standard-lib-deprecated-specifically
+def deprecated(func: TFunc) -> TFunc:
+    """This is a decorator which can be used to mark functions
+    as deprecated. It will result in a warning being emitted
+    when the function is used."""
+    @functools.wraps(func)
+    def new_func(*args, **kwargs):  # type: ignore
+        warnings.simplefilter('always', DeprecationWarning)  # turn off filter
+        warnings.warn("Call to deprecated function {}.".format(func.__name__), category=DeprecationWarning, stacklevel=2)
+        warnings.simplefilter('default', DeprecationWarning)  # reset filter
+        return func(*args, **kwargs)
+    return cast(TFunc, new_func)

ul_api_utils/utils/flags.py

@@ -0,0 +1,29 @@
+import functools
+import hashlib
+
+
+class Flagged:
+    def __init__(self, flags: str) -> None:
+        self._flsgs = set(_feature.strip() for _feature in flags.strip().upper().split(',') if len(_feature.strip()))
+
+    @functools.cache  # noqa: B019
+    def has_or_unset(self, flag: str) -> bool:
+        flags = set(_feature.strip() for _feature in flag.strip().upper().split(',') if len(_feature.strip()))
+        if not len(flags):
+            return True
+        for f in flags:
+            if f in self._flsgs:
+                return True
+            fh = hashlib.sha1()
+            fh.update(f.encode())
+            if fh.digest().hex().upper() in self._flsgs:
+                return True
+        return False
+
+    def gen_hashed(self) -> str:
+        res_flags = []
+        for f in self._flsgs:
+            fh = hashlib.sha1()
+            fh.update(f.encode())
+            res_flags.append(fh.digest().hex().upper())
+        return ",".join(res_flags)

ul_api_utils/utils/flask_swagger_generator/conf.py

@@ -0,0 +1,4 @@
+from typing import Tuple
+
+TVersion = Tuple[int, int, int, str, int]
+VERSION: TVersion = (0, 3, 1, 'alpha', 0)

ul_api_utils/utils/flask_swagger_generator/exceptions.py

@@ -0,0 +1,7 @@
+from typing import Optional
+
+
+class SwaggerGeneratorError(Exception):
+
+    def __init__(self, message: Optional[str] = None) -> None:
+        super().__init__(message)

ul_api_utils/utils/flask_swagger_generator/specifiers/swagger_models.py

@@ -0,0 +1,57 @@
+from abc import ABC, abstractmethod
+from random import randint
+
+
+class SwaggerModel(ABC):
+    TAB = "  "
+
+    def __init__(self) -> None:
+        self.swagger_models = []  # type: ignore
+
+    def add_swagger_model(self, child_swagger_model):  # type: ignore
+        self.swagger_models.append(child_swagger_model)
+
+    def add_swagger_models(self, child_swagger_models):  # type: ignore
+        self.swagger_models += child_swagger_models
+
+    def has_swagger_model_child_type(self, child_swagger_model_type) -> bool:  # type: ignore
+        for swagger_model in self.swagger_models:
+
+            if isinstance(swagger_model, child_swagger_model_type):
+                return True
+
+        return False
+
+    def get_swagger_child_models_of_type(self, swagger_model_type):  # type: ignore
+        selection = []
+
+        for swagger_model in self.swagger_models:
+
+            if isinstance(swagger_model, swagger_model_type):
+                selection.append(swagger_model)
+
+        return selection
+
+    def write(self, file) -> None:  # type: ignore
+        self.perform_write(file)
+
+        for swagger_model in self.swagger_models:
+            swagger_model.write(file)
+
+    @staticmethod
+    def indent(string: str, prefix: str) -> str:
+        return ''.join(prefix + line for line in string.splitlines(True))
+
+    @abstractmethod
+    def perform_write(self, file) -> None:  # type: ignore
+        raise NotImplementedError()
+
+    def random_id(self) -> int:
+        """
+        Function to create a random ID.
+        Returns: random integer that can be used as an ID
+        """
+        minimal = 100
+        maximal = 1000000000
+        rand = randint(minimal, maximal)
+        return rand

ul_api_utils/utils/flask_swagger_generator/specifiers/swagger_specifier.py

@@ -0,0 +1,48 @@
+from abc import ABC, abstractmethod
+from typing import List, Optional
+
+from ul_api_utils.utils.flask_swagger_generator.utils.request_type import RequestType
+from ul_api_utils.utils.flask_swagger_generator.utils.security_type import SecurityType
+
+
+class SwaggerSpecifier(ABC):
+
+    def __init__(self) -> None:
+        self.application_name: Optional[str] = None
+        self.application_version: Optional[str] = None
+
+    @abstractmethod
+    def add_response(self, function_name: str, status_code: int, schema, description: str = ""):  # type: ignore
+        raise NotImplementedError()
+
+    @abstractmethod
+    def add_endpoint(
+        self,
+        function_name: str,
+        path: str,
+        request_types: List[RequestType],
+        group: Optional[str] = None,
+    ) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def add_query_parameters(self) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def add_request_body(self, function_name: str, schema) -> None:  # type: ignore
+        raise NotImplementedError()
+
+    @abstractmethod
+    def add_security(self, function_name: str, security_type: SecurityType) -> None:
+        raise NotImplementedError()
+
+    def set_application_name(self, application_name: str) -> None:
+        self.application_name = application_name
+
+    def set_application_version(self, application_version: str) -> None:
+        self.application_version = application_version
+
+    @abstractmethod
+    def clean(self) -> None:
+        pass