opengris-scaler 1.12.37__cp38-cp38-musllinux_1_2_x86_64.whl

scaler/utility/logging/utility.py

@@ -0,0 +1,183 @@
+import dataclasses
+import enum
+import logging
+import logging.config
+import logging.handlers
+import os
+import typing
+
+from scaler.config.defaults import DEFAULT_LOGGING_PATHS
+
+
+class LogType(enum.Enum):
+    Screen = enum.auto()
+    File = enum.auto()
+
+
+@dataclasses.dataclass
+class LogPath:
+    log_type: LogType
+    path: str
+
+
+class LoggingLevel(enum.Enum):
+    CRITICAL = logging.CRITICAL
+    ERROR = logging.ERROR
+    WARNING = logging.WARNING
+    INFO = logging.INFO
+    DEBUG = logging.DEBUG
+    NOTSET = logging.NOTSET
+
+
+def setup_logger(
+    log_paths: typing.Tuple[str, ...] = DEFAULT_LOGGING_PATHS,
+    logging_config_file: typing.Optional[str] = None,
+    logging_level: str = LoggingLevel.INFO.name,
+):
+    if not log_paths and not logging_config_file:
+        return
+
+    if isinstance(log_paths, str):
+        log_paths = (log_paths,)
+
+    if logging_config_file is not None:
+        print(f"use logging config file: {logging_config_file}")
+        logging.config.fileConfig(logging_config_file, disable_existing_loggers=True)
+        return
+
+    resolved_log_paths = [LogPath(log_type=__detect_log_types(file_name), path=file_name) for file_name in log_paths]
+    __logging_config(log_paths=resolved_log_paths, logging_level=logging_level)
+    logging.info(f"logging to {log_paths}")
+
+
+def __detect_log_types(file_name: str) -> LogType:
+    if file_name in {"-", "/dev/stdout"}:
+        return LogType.Screen
+
+    return LogType.File
+
+
+def __format(name) -> str:
+    if not name:
+        return ""
+
+    return "%({name})s".format(name=name)
+
+
+def __generate_log_config() -> typing.Dict:
+    return {
+        "version": 1,
+        "disable_existing_loggers": False,  # this fixes the problem
+        "formatters": {
+            "standard": {
+                "format": "[{levelname}]{asctime}: {message}".format(
+                    levelname=__format("levelname"), asctime=__format("asctime"), message=__format("message")
+                ),
+                "datefmt": "%Y-%m-%d %H:%M:%S%z",
+            },
+            "verbose": {
+                "format": "[{levelname}]{asctime}:{module}:{funcName}:{lineno}: {message}".format(
+                    levelname=__format("levelname"),
+                    asctime=__format("asctime"),
+                    module=__format("module"),
+                    funcName=__format("funcName"),
+                    lineno=__format("lineno"),
+                    message=__format("message"),
+                ),
+                "datefmt": "%Y-%m-%d %H:%M:%S%z",
+            },
+        },
+        "handlers": {},
+        "loggers": {"": {"handlers": [], "level": "DEBUG", "propagate": True}},
+    }
+
+
+def __logging_config(log_paths: typing.List[LogPath], logging_level: str = LoggingLevel.INFO.name):
+    logging.addLevelName(logging.INFO, "INFO")
+    logging.addLevelName(logging.WARNING, "WARN")
+    logging.addLevelName(logging.ERROR, "EROR")
+    logging.addLevelName(logging.DEBUG, "DEBG")
+    logging.addLevelName(logging.CRITICAL, "CTIC")
+
+    config = __generate_log_config()
+    handlers = config["handlers"]
+    root_loggers = config["loggers"][""]["handlers"]
+
+    for log_path in log_paths:
+        if log_path.log_type == LogType.Screen:
+            handlers["console"] = __create_stdout_handler(logging_level)
+            root_loggers.append("console")
+            continue
+
+        elif log_path.log_type == LogType.File:
+            handlers[log_path.path] = __create_time_rotating_file_handler(logging_level, log_path.path)
+            root_loggers.append(log_path.path)
+            continue
+
+        raise TypeError(f"Unsupported LogPath: {log_path}")
+
+    logging.config.dictConfig(config)
+
+
+def __create_stdout_handler(logging_level: str):
+    return {
+        "class": "logging.StreamHandler",
+        "level": logging_level,
+        "formatter": "standard",
+        "stream": "ext://sys.stdout",
+    }
+
+
+def __create_time_rotating_file_handler(logging_level: str, file_path: str):
+    return {
+        "class": "logging.handlers.TimedRotatingFileHandler",
+        "level": logging_level,
+        "formatter": "verbose",
+        "filename": os.path.expandvars(os.path.expanduser(file_path)),
+        "when": "midnight",
+    }
+
+
+def __create_size_rotating_file_handler(log_path) -> typing.Dict:
+    return {
+        "class": "logging.handlers.RotatingFileHandler",
+        "level": "INFO",
+        "formatter": "verbose",
+        "filename": os.path.expandvars(os.path.expanduser(log_path)),
+        "maxBytes": 10485760,
+        "backupCount": 20,
+        "encoding": "utf8",
+    }
+
+
+def __parse_logging_level(value):
+    return LoggingLevel(value).value
+
+
+def get_logger_info(logger: logging.Logger) -> typing.Tuple[str, str, typing.Tuple[str, ...]]:
+    """
+    Retrieves the format string, level string, and all active log paths from a logger's handlers.
+    """
+    log_level_str = logging.getLevelName(logger.getEffectiveLevel())
+    log_format_str = ""
+    log_paths: typing.List[str] = []
+
+    if logger.hasHandlers():
+        first_handler = logger.handlers[0]
+        if first_handler.formatter:
+            log_format_str = getattr(first_handler.formatter, "_fmt", "")
+
+        for handler in logger.handlers:
+            if isinstance(handler, logging.handlers.BaseRotatingHandler):
+                log_paths.append(handler.baseFilename)
+            elif isinstance(handler, logging.StreamHandler) and hasattr(handler.stream, "name"):
+                if "stdout" in handler.stream.name:
+                    log_paths.append("/dev/stdout")
+                elif "stderr" in handler.stream.name:
+                    log_paths.append("/dev/stderr")
+
+    # If no specific path was found, default to stdout
+    if not log_paths:
+        log_paths.append("/dev/stdout")
+
+    return log_format_str, log_level_str, tuple(log_paths)

scaler/utility/many_to_many_dict.py

@@ -0,0 +1,123 @@
+from typing import Dict, Generic, Iterable, Set, Tuple, TypeVar
+
+KeyT = TypeVar("KeyT")
+ValueT = TypeVar("ValueT")
+
+LeftKeyT = TypeVar("LeftKeyT")
+RightKeyT = TypeVar("RightKeyT")
+
+
+class ManyToManyDict(Generic[LeftKeyT, RightKeyT]):
+    def __init__(self):
+        self._left_key_to_right_key_set: _KeyValueDictSet[LeftKeyT, RightKeyT] = _KeyValueDictSet()
+        self._right_key_to_left_key_set: _KeyValueDictSet[RightKeyT, LeftKeyT] = _KeyValueDictSet()
+
+    def left_keys(self):
+        return self._left_key_to_right_key_set.keys()
+
+    def right_keys(self):
+        return self._right_key_to_left_key_set.keys()
+
+    def add(self, left_key: LeftKeyT, right_key: RightKeyT):
+        self._left_key_to_right_key_set.add(left_key, right_key)
+        self._right_key_to_left_key_set.add(right_key, left_key)
+
+    def remove(self, left_key: LeftKeyT, right_key: RightKeyT):
+        self._left_key_to_right_key_set.remove_value(left_key, right_key)
+        self._right_key_to_left_key_set.remove_value(right_key, left_key)
+
+    def has_left_key(self, left_key: LeftKeyT) -> bool:
+        return left_key in self._left_key_to_right_key_set
+
+    def has_right_key(self, right_key: RightKeyT) -> bool:
+        return right_key in self._right_key_to_left_key_set
+
+    def has_key_pair(self, left_key: LeftKeyT, right_key: RightKeyT) -> bool:
+        return (
+            self.has_left_key(left_key)
+            and self.has_right_key(right_key)
+            and right_key in self._left_key_to_right_key_set.get_values(left_key)
+            and left_key in self._right_key_to_left_key_set.get_values(right_key)
+        )
+
+    def left_key_items(self) -> Iterable[Tuple[LeftKeyT, Set[RightKeyT]]]:
+        return self._left_key_to_right_key_set.items()
+
+    def right_key_items(self) -> Iterable[Tuple[RightKeyT, Set[LeftKeyT]]]:
+        return self._right_key_to_left_key_set.items()
+
+    def get_left_items(self, right_key: RightKeyT) -> Set[LeftKeyT]:
+        if right_key not in self._right_key_to_left_key_set:
+            raise ValueError(f"cannot find {right_key=} in ManyToManyDict")
+
+        return self._right_key_to_left_key_set.get_values(right_key)
+
+    def get_right_items(self, left_key: LeftKeyT) -> Set[RightKeyT]:
+        if left_key not in self._left_key_to_right_key_set:
+            raise ValueError(f"cannot find {left_key=} in ManyToManyDict")
+
+        return self._left_key_to_right_key_set.get_values(left_key)
+
+    def remove_left_key(self, left_key: LeftKeyT) -> Set[RightKeyT]:
+        if left_key not in self._left_key_to_right_key_set:
+            raise KeyError(f"cannot find {left_key=} in ManyToManyDict")
+
+        right_keys = self._left_key_to_right_key_set.remove_key(left_key)
+        for right_key in right_keys:
+            self._right_key_to_left_key_set.remove_value(right_key, left_key)
+
+        return right_keys
+
+    def remove_right_key(self, right_key: RightKeyT) -> Set[LeftKeyT]:
+        if right_key not in self._right_key_to_left_key_set:
+            raise ValueError(f"cannot find {right_key=} in ManyToManyDict")
+
+        left_keys = self._right_key_to_left_key_set.remove_key(right_key)
+        for left_key in left_keys:
+            self._left_key_to_right_key_set.remove_value(left_key, right_key)
+
+        return left_keys
+
+
+class _KeyValueDictSet(Generic[KeyT, ValueT]):
+    def __init__(self):
+        self._key_to_value_set: Dict[KeyT, Set[ValueT]] = dict()
+
+    def __contains__(self, key) -> bool:
+        return key in self._key_to_value_set
+
+    def keys(self):
+        return self._key_to_value_set.keys()
+
+    def values(self):
+        return self._key_to_value_set.values()
+
+    def items(self):
+        return self._key_to_value_set.items()
+
+    def add(self, key: KeyT, value: ValueT):
+        if key not in self._key_to_value_set:
+            self._key_to_value_set[key] = set()
+
+        self._key_to_value_set[key].add(value)
+
+    def get_values(self, key: KeyT) -> Set[ValueT]:
+        if key not in self._key_to_value_set:
+            raise ValueError(f"cannot find {key=} in KeyValueSet")
+
+        return self._key_to_value_set[key]
+
+    def remove_key(self, key: KeyT) -> Set[ValueT]:
+        if key not in self._key_to_value_set:
+            raise KeyError(f"cannot find {key=} in KeyValueSet")
+
+        values = self._key_to_value_set.pop(key)
+        return values
+
+    def remove_value(self, key: KeyT, value: ValueT):
+        if key not in self._key_to_value_set:
+            raise KeyError(f"cannot find {key=} in KeyValueSet")
+
+        self._key_to_value_set[key].remove(value)
+        if not self._key_to_value_set[key]:
+            self._key_to_value_set.pop(key)

scaler/utility/metadata/profile_result.py

@@ -0,0 +1,31 @@
+import dataclasses
+import struct
+from typing import Optional
+
+from scaler.protocol.python.message import TaskResult
+
+
+@dataclasses.dataclass
+class ProfileResult:
+    duration_s: float = dataclasses.field(default=0.0)
+    memory_peak: int = dataclasses.field(default=0)
+    cpu_time_s: float = dataclasses.field(default=0.0)
+
+    FORMAT = "!fQf"  # duration, memory peak, CPU time
+
+    def serialize(self) -> bytes:
+        return struct.pack(self.FORMAT, self.duration_s, self.memory_peak, self.cpu_time_s)
+
+    @staticmethod
+    def deserialize(data: bytes) -> "ProfileResult":
+        return ProfileResult(*struct.unpack(ProfileResult.FORMAT, data))
+
+
+def retrieve_profiling_result_from_task_result(task_result: TaskResult) -> Optional[ProfileResult]:
+    if task_result.metadata == b"":
+        return None
+
+    try:
+        return ProfileResult.deserialize(task_result.metadata)
+    except struct.error:
+        raise ValueError(f"unexpected metadata value (expected {ProfileResult.__name__}).")

scaler/utility/metadata/task_flags.py

@@ -0,0 +1,30 @@
+import dataclasses
+import struct
+
+from scaler.protocol.python.message import Task
+
+
+@dataclasses.dataclass
+class TaskFlags:
+    profiling: bool = dataclasses.field(default=True)
+    priority: int = dataclasses.field(default=0)
+    stream_output: bool = dataclasses.field(default=False)
+
+    FORMAT = "!?i?"
+
+    def serialize(self) -> bytes:
+        return struct.pack(TaskFlags.FORMAT, self.profiling, self.priority, self.stream_output)
+
+    @staticmethod
+    def deserialize(data: bytes) -> "TaskFlags":
+        return TaskFlags(*struct.unpack(TaskFlags.FORMAT, data))
+
+
+def retrieve_task_flags_from_task(task: Task) -> TaskFlags:
+    if task.metadata == b"":
+        return TaskFlags()
+
+    try:
+        return TaskFlags.deserialize(task.metadata)
+    except struct.error:
+        raise ValueError(f"unexpected metadata value (expected {TaskFlags.__name__}).")

scaler/utility/mixins.py

@@ -0,0 +1,13 @@
+import abc
+
+
+class Looper(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    async def routine(self):
+        raise NotImplementedError()
+
+
+class Reporter(metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def get_status(self):
+        raise NotImplementedError()

scaler/utility/network_util.py

@@ -0,0 +1,7 @@
+import socket
+
+
+def get_available_tcp_port(hostname: str = "127.0.0.1") -> int:
+    with socket.socket(socket.AddressFamily.AF_INET, socket.SocketKind.SOCK_STREAM) as sock:
+        sock.bind((hostname, 0))
+        return sock.getsockname()[1]

scaler/utility/one_to_many_dict.py

@@ -0,0 +1,72 @@
+from typing import Dict, Generic, Set, TypeVar
+
+KeyT = TypeVar("KeyT")
+ValueT = TypeVar("ValueT")
+
+
+class OneToManyDict(Generic[KeyT, ValueT]):
+    def __init__(self):
+        self._key_to_value_set: Dict[KeyT, Set[ValueT]] = dict()
+        self._value_to_key: Dict[ValueT, KeyT] = dict()
+
+    def __contains__(self, key) -> bool:
+        return self.has_key(key)
+
+    def keys(self):
+        return self._key_to_value_set.keys()
+
+    def values(self):
+        return self._key_to_value_set.values()
+
+    def items(self):
+        return self._key_to_value_set.items()
+
+    def add(self, key: KeyT, value: ValueT):
+        if value in self._value_to_key and self._value_to_key[value] != key:
+            raise ValueError("value has to be unique in OneToManyDict")
+
+        self._value_to_key[value] = key
+
+        if key not in self._key_to_value_set:
+            self._key_to_value_set[key] = set()
+
+        self._key_to_value_set[key].add(value)
+
+    def has_key(self, key: KeyT) -> bool:
+        return key in self._key_to_value_set
+
+    def has_value(self, value: ValueT) -> bool:
+        return value in self._value_to_key
+
+    def get_key(self, value: ValueT) -> KeyT:
+        if value not in self._value_to_key:
+            raise ValueError(f"cannot find {value=} in OneToManyDict")
+
+        return self._value_to_key[value]
+
+    def get_values(self, key: KeyT) -> Set[ValueT]:
+        if key not in self._key_to_value_set:
+            raise ValueError(f"cannot find {key=} in OneToManyDict")
+
+        return self._key_to_value_set[key]
+
+    def remove_key(self, key: KeyT) -> Set[ValueT]:
+        if key not in self._key_to_value_set:
+            raise KeyError(f"cannot find {key=} in OneToManyDict")
+
+        values = self._key_to_value_set.pop(key)
+        for value in values:
+            self._value_to_key.pop(value)
+
+        return values
+
+    def remove_value(self, value: ValueT) -> KeyT:
+        if value not in self._value_to_key:
+            raise ValueError(f"cannot find {value=} in OneToManyDict")
+
+        key = self._value_to_key.pop(value)
+        self._key_to_value_set[key].remove(value)
+        if not self._key_to_value_set[key]:
+            self._key_to_value_set.pop(key)
+
+        return key

scaler/utility/queues/async_indexed_queue.py

@@ -0,0 +1,37 @@
+from asyncio import Queue, QueueEmpty
+from typing import Generic, TypeVar
+
+from scaler.utility.queues.indexed_queue import IndexedQueue
+
+ItemType = TypeVar("ItemType")
+
+
+class AsyncIndexedQueue(Queue, Generic[ItemType]):
+    """This should have same set of features as asyncio.Queue, with additional methods like remove
+    - it behaves like regular async queue, except:
+      - all the items pushed to queue should be hashable
+      - those items should be unique in queue
+    - IndexedQueue.put(), IndexedQueue.get(), IndexedQueue.remove() should all take O(1) time complexity
+    """
+
+    def __contains__(self, item: ItemType):
+        return item in self._queue
+
+    def __len__(self):
+        return self._queue.__len__()
+
+    def _init(self, maxsize):
+        self._queue = IndexedQueue()
+
+    def _put(self, item: ItemType):
+        self._queue.put(item)
+
+    def _get(self):
+        try:
+            return self._queue.get()
+        except IndexError:
+            raise QueueEmpty(f"{self.__class__.__name__} queue empty")
+
+    def remove(self, item: ItemType):
+        """remove the item in the queue in O(1) time complexity"""
+        self._queue.remove(item)

scaler/utility/queues/async_priority_queue.py

@@ -0,0 +1,70 @@
+import heapq
+import sys
+from asyncio import Queue
+from typing import Any, Dict, List, Tuple, Union
+
+PriorityType = Union[int, Tuple["PriorityType", ...]]
+
+
+class AsyncPriorityQueue(Queue):
+    """A subclass of Queue; retrieves entries in priority order (lowest first).
+
+    Entries are typically list of the form: [priority, data].
+    """
+
+    def __len__(self):
+        return len(self._queue)
+
+    def _init(self, maxsize):
+        self._queue: List[List] = []
+        self._locator: Dict[bytes, List] = {}
+
+    def _put(self, item):
+        if not isinstance(item, list):
+            item = list(item)
+
+        heapq.heappush(self._queue, item)
+        self._locator[item[1]] = item
+
+    def _get(self):
+        priority, data = heapq.heappop(self._queue)
+        self._locator.pop(data)
+        return priority, data
+
+    def remove(self, data):
+        # this operation is O(n), first change priority to -1 and pop from top of the heap, mark it as invalid
+        # entry in the heap is not good idea as those invalid, entry will never get removed, so we used heapq internal
+        # function _siftdown to maintain min heap invariant
+        item = self._locator.pop(data)
+        i = self._queue.index(item)  # O(n)
+        item[0] = self.__to_lowest_priority(item[0])
+        heapq._siftdown(self._queue, 0, i)  # type: ignore[attr-defined]
+        assert heapq.heappop(self._queue) == item
+
+    def decrease_priority(self, data):
+        # this operation should be O(n), mark it as invalid entry in the heap is not good idea as those invalid
+        # entry will never get removed, so we used heapq internal function _siftdown to maintain min heap invariant
+        item = self._locator[data]
+        i = self._queue.index(item)  # O(n)
+        item[0] = self.__to_lower_priority(item[0])
+        heapq._siftdown(self._queue, 0, i)  # type: ignore[attr-defined]
+
+    def max_priority_item(self) -> Tuple[PriorityType, Any]:
+        """output the Tuple of top priority number and top priority item"""
+        item = heapq.heappop(self._queue)
+        heapq.heappush(self._queue, item)
+        return item[0], item[1]
+
+    @classmethod
+    def __to_lowest_priority(cls, original_priority: PriorityType) -> PriorityType:
+        if isinstance(original_priority, tuple):
+            return tuple(cls.__to_lowest_priority(value) for value in original_priority)
+        else:
+            return -sys.maxsize - 1
+
+    @classmethod
+    def __to_lower_priority(cls, original_priority: PriorityType) -> PriorityType:
+        if isinstance(original_priority, tuple):
+            return tuple(cls.__to_lower_priority(value) for value in original_priority)
+        else:
+            return original_priority - 1

scaler/utility/queues/async_sorted_priority_queue.py

@@ -0,0 +1,45 @@
+from asyncio import Queue
+from typing import Any, Dict
+
+from scaler.utility.queues.async_priority_queue import AsyncPriorityQueue
+
+
+class AsyncSortedPriorityQueue(Queue):
+    """A subclass of Queue; retrieves entries in priority order (lowest first), and then by adding order.
+
+    Entries are typically list of the form: [priority number, data].
+    """
+
+    def __len__(self):
+        return len(self._queue)
+
+    def _init(self, maxsize: int):
+        self._queue = AsyncPriorityQueue()
+
+        # Keeps an item count to assign monotonic integer to queued items, so to also keep the priority queue sorted by
+        # adding order.
+        # See https://docs.python.org/3/library/heapq.html#priority-queue-implementation-notes.
+        self._item_counter: int = 0
+        self._data_to_item_id: Dict[Any, int] = dict()
+
+    def _put(self, item) -> None:
+        priority, data = item
+
+        if data in self._data_to_item_id:
+            raise ValueError(f"item `{data}` already in the queue")
+
+        item_id = self._item_counter
+        self._item_counter += 1
+
+        self._queue._put([priority, (item_id, data)])
+        self._data_to_item_id[data] = item_id
+
+    def _get(self):
+        priority, (_, data) = self._queue._get()
+        self._data_to_item_id.pop(data)
+
+        return [priority, data]
+
+    def remove(self, data: Any) -> None:
+        item_id = self._data_to_item_id.pop(data)
+        self._queue.remove((item_id, data))