opengris-scaler 1.12.37__cp38-cp38-musllinux_1_2_x86_64.whl

scaler/utility/event_loop.py

@@ -0,0 +1,58 @@
+import asyncio
+import enum
+import logging
+from typing import Awaitable, Callable
+
+
+class EventLoopType(enum.Enum):
+    builtin = enum.auto()
+    uvloop = enum.auto()
+
+    @staticmethod
+    def allowed_types():
+        return {m.name for m in EventLoopType}
+
+
+def register_event_loop(event_loop_type: str):
+    if event_loop_type not in EventLoopType.allowed_types():
+        raise TypeError(f"allowed event loop types are: {EventLoopType.allowed_types()}")
+
+    event_loop_type_enum = EventLoopType[event_loop_type]
+    if event_loop_type_enum == EventLoopType.uvloop:
+        try:
+            import uvloop  # noqa
+        except ImportError:
+            raise ImportError("please use pip install uvloop if try to use uvloop as event loop")
+
+        uvloop.install()
+
+    assert event_loop_type in EventLoopType.allowed_types()
+
+    logging.info(f"use event loop: {event_loop_type}")
+
+
+def create_async_loop_routine(routine: Callable[[], Awaitable], seconds: int):
+    """create async loop routine,
+
+    - if seconds is negative, means disable
+    - 0 means looping without any wait, as fast as possible
+    - positive number means execute routine every positive seconds, if passing 1 means run once every 1 seconds"""
+
+    async def loop():
+        if seconds < 0:
+            logging.info(f"{routine.__self__.__class__.__name__}: disabled")  # type: ignore[attr-defined]
+            return
+
+        logging.info(f"{routine.__self__.__class__.__name__}: started")  # type: ignore[attr-defined]
+        try:
+            while True:
+                await routine()
+                await asyncio.sleep(seconds)
+        except asyncio.CancelledError:
+            pass
+        except KeyboardInterrupt:
+            pass
+
+        logging.info(f"{routine.__self__.__class__.__name__}: exited")  # type: ignore[attr-defined]
+
+    return loop()

scaler/utility/exceptions.py

@@ -0,0 +1,42 @@
+class TaskNotFoundError(Exception):
+    pass
+
+
+class WorkerDiedError(Exception):
+    pass
+
+
+class NoWorkerError(Exception):
+    pass
+
+
+class DisconnectedError(Exception):
+    pass
+
+
+class ProcessorDiedError(Exception):
+    pass
+
+
+class DeserializeObjectError(Exception):
+    pass
+
+
+class MissingObjects(Exception):
+    pass
+
+
+class ClientCancelledException(Exception):
+    pass
+
+
+class ClientShutdownException(Exception):
+    pass
+
+
+class ClientQuitException(Exception):
+    pass
+
+
+class ObjectStorageException(Exception):
+    pass

scaler/utility/formatter.py

@@ -0,0 +1,44 @@
+STORAGE_SIZE_MODULUS = 1024.0
+TIME_MODULUS = 1000
+
+
+def format_bytes(number) -> str:
+    for unit in ["B", "K", "M", "G", "T"]:
+        if number >= STORAGE_SIZE_MODULUS:
+            number /= STORAGE_SIZE_MODULUS
+            continue
+
+        if unit in {"B", "K"}:
+            return f"{int(number)}{unit}"
+
+        return f"{number:.1f}{unit}"
+
+    raise ValueError("This should not happen")
+
+
+def format_integer(number):
+    return f"{number:,}"
+
+
+def format_percentage(number: int):
+    return f"{(number/1000):.1%}"
+
+
+def format_microseconds(number: int):
+    for unit in ["us", "ms", "s"]:
+        if number >= TIME_MODULUS:
+            number = int(number / TIME_MODULUS)
+            continue
+
+        if unit == "us":
+            return f"{number/TIME_MODULUS:.1f}ms"
+
+        too_big_sign = "+" if unit == "s" and number > TIME_MODULUS else ""
+        return f"{int(number)}{too_big_sign}{unit}"
+
+
+def format_seconds(number: int):
+    if number > 60:
+        return "60+s"
+
+    return f"{number}s"

scaler/utility/graph/optimization.py

@@ -0,0 +1,27 @@
+from collections import deque
+from typing import Any, Callable, Dict, List, Tuple, Union
+
+
+def cull_graph(
+    graph: Dict[str, Tuple[Union[Callable, Any], ...]], keys: List[str]
+) -> Dict[str, Tuple[Union[Callable, Any], ...]]:
+    queue = deque(keys)
+    visited = set()
+    for target_key in keys:
+        visited.add(target_key)
+
+    while queue:
+        key = queue.popleft()
+
+        task = graph[key]
+        if not (isinstance(task, tuple) and task and callable(task[0])):
+            continue
+
+        dependencies = set(task[1:])
+        for predecessor_key in dependencies:
+            if predecessor_key in visited:
+                continue
+            visited.add(predecessor_key)
+            queue.append(predecessor_key)
+
+    return {key: graph[key] for key in visited}

scaler/utility/graph/topological_sorter.py

@@ -0,0 +1,11 @@
+import logging
+
+try:
+    from scaler.utility.graph.topological_sorter_graphblas import TopologicalSorter
+
+    logging.info("using GraphBLAS for calculate graph")
+except ImportError as e:
+    assert isinstance(e, Exception)
+    from graphlib import TopologicalSorter  # type: ignore[assignment, no-redef]
+
+    assert isinstance(TopologicalSorter, object)

scaler/utility/graph/topological_sorter_graphblas.py

@@ -0,0 +1,174 @@
+import collections
+import graphlib
+import itertools
+from typing import Generic, Hashable, Iterable, List, Mapping, Optional, Tuple, TypeVar
+
+from bidict import bidict
+
+try:
+    import graphblas as gb
+    import numpy as np  # noqa
+except ImportError:
+    raise ImportError("Please use 'pip install python-graphblas' to have graph blas support")
+
+GraphKeyType = TypeVar("GraphKeyType", bound=Hashable)
+
+
+class TopologicalSorter(Generic[GraphKeyType]):
+    """
+    Implements graphlib's TopologicalSorter, but the graph handling is backed by GraphBLAS
+    Reference: https://github.com/python/cpython/blob/4a3ea1fdd890e5e2ec26540dc3c958a52fba6556/Lib/graphlib.py
+    """
+
+    def __init__(self, graph: Optional[Mapping[GraphKeyType, Iterable[GraphKeyType]]] = None):
+        # the layout of the matrix is (in-vertex, out-vertex)
+        self._matrix = gb.Matrix(gb.dtypes.BOOL)
+        self._key_to_id: bidict[GraphKeyType, int] = bidict()
+
+        self._graph_matrix_mask: Optional[np.ndarray] = None
+        self._visited_vertices_mask: Optional[np.ndarray] = None
+        self._ready_nodes: Optional[List[GraphKeyType]] = None
+
+        self._n_done = 0
+        self._n_visited = 0
+
+        if graph is not None:
+            self.merge_graph(graph)
+
+    def add(self, node: GraphKeyType, *predecessors: GraphKeyType) -> None:
+        self.merge_graph({node: predecessors})
+
+    def merge_graph(self, graph: Mapping[GraphKeyType, Iterable[GraphKeyType]]) -> None:
+        if self._ready_nodes is not None:
+            raise ValueError("nodes cannot be added after a call to prepare()")
+
+        # cache old dim to compare later when resizing matrix
+        old_dim = len(self._key_to_id)
+
+        # maintain iterable copies for iterable predecessors
+        graph_iterable_copy = {}
+
+        # update key to id mappings
+        for node, predecessors in graph.items():
+            if node not in self._key_to_id:
+                self._key_to_id[node] = len(self._key_to_id)
+
+            # copy iterator if predecessors is an iterable
+            if isinstance(predecessors, collections.abc.Iterable):
+                predecessors, graph_iterable_copy[node] = itertools.tee(predecessors)
+
+            for pred in predecessors:
+                if pred not in self._key_to_id:
+                    self._key_to_id[pred] = len(self._key_to_id)
+
+        # resize at once as it is faster
+        if old_dim != len(self._key_to_id):
+            self._matrix.resize(len(self._key_to_id), len(self._key_to_id))
+
+        # update matrix
+        for node, predecessors in graph.items():
+            if node in graph_iterable_copy:
+                predecessors = graph_iterable_copy[node]
+
+            for pred in predecessors:
+                self._matrix[self._key_to_id[node], self._key_to_id[pred]] = True
+
+    def prepare(self) -> None:
+        if self._ready_nodes is not None:
+            raise ValueError("cannot prepare() more than once")
+
+        self._graph_matrix_mask = np.ones(len(self._key_to_id), bool)
+        self._visited_vertices_mask = np.zeros(len(self._key_to_id), bool)
+
+        self._ready_nodes = self._get_zero_degree_keys()
+        for node in self._ready_nodes:
+            self._visited_vertices_mask[self._key_to_id[node]] = True
+        self._n_visited += len(self._ready_nodes)
+
+        if self._has_cycle():
+            raise graphlib.CycleError("cycle detected")
+
+    def get_ready(self) -> Tuple[GraphKeyType, ...]:
+        if self._ready_nodes is None:
+            raise ValueError("prepare() must be called first")
+
+        result = tuple(self._ready_nodes)
+        self._ready_nodes.clear()
+        return result
+
+    def is_active(self) -> bool:
+        if self._ready_nodes is None:
+            raise ValueError("prepare() must be called first")
+        return self._n_done < self._n_visited or bool(self._ready_nodes)
+
+    def __bool__(self) -> bool:
+        return self.is_active()
+
+    def done(self, *nodes: GraphKeyType) -> None:
+        if self._ready_nodes is None:
+            raise ValueError("prepare() must be called first")
+
+        for node in nodes:
+            if node not in self._key_to_id:
+                raise ValueError(f"node {node!r} was not added using add()")
+
+            _id = self._key_to_id[node]
+
+            if not self._visited_vertices_mask[_id]:
+                raise ValueError(f"node {node!r} is not ready")
+
+            if not self._graph_matrix_mask[_id]:
+                raise ValueError(f"node {node!r} is already done")
+
+            self._graph_matrix_mask[_id] = False
+        self._n_done += len(nodes)
+
+        new_ready_nodes = self._get_zero_degree_keys()
+        for node in new_ready_nodes:
+            self._visited_vertices_mask[self._key_to_id[node]] = True
+        self._ready_nodes.extend(new_ready_nodes)
+        self._n_visited += len(new_ready_nodes)
+
+    def static_order(self) -> Iterable[GraphKeyType]:
+        self.prepare()
+        while self.is_active():
+            node_group = self.get_ready()
+            yield from node_group
+            self.done(*node_group)
+
+    def _has_cycle(self) -> bool:
+        """
+        Detect cycle using trace(A^n) != 0.
+        https://arxiv.org/pdf/1610.01200.pdf
+
+        :return: True if cycle is found, otherwise False
+        """
+        matrix_n = gb.Vector.from_dense(np.ones(len(self._key_to_id), bool), missing_value=False).diag()
+        for _ in range(len(self._key_to_id)):
+            # use LOR_PAIR to compute matrix multiplication over boolean matrices
+            matrix_n << gb.semiring.lor_pair(matrix_n @ self._matrix)
+            # check diagonal for any truthy values
+            if matrix_n.diag().reduce(gb.monoid.lor):
+                return True
+        return False
+
+    def _get_zero_degree_keys(self) -> List[GraphKeyType]:
+        ids = self._get_mask_diff(self._visited_vertices_mask, self._get_zero_degree_mask(self._get_masked_matrix()))
+        return [self._key_to_id.inverse[_id] for _id in ids]
+
+    def _get_masked_matrix(self) -> gb.Matrix:
+        # convert vector mask to matrix diagonal and then perform matrix multiplication to mask matrix
+        # https://github.com/DrTimothyAldenDavis/GraphBLAS/issues/48#issuecomment-858596341
+        return gb.semiring.lor_pair(
+            self._matrix @ gb.Vector.from_dense(self._graph_matrix_mask, missing_value=False).diag()
+        )
+
+    @classmethod
+    def _get_zero_degree_mask(cls, masked_matrix: gb.Matrix) -> np.ndarray:
+        degrees = masked_matrix.reduce_rowwise(gb.monoid.lor)
+        indices, _ = degrees.to_coo(indices=True, values=False, sort=False)
+        return np.logical_not(np.in1d(np.arange(masked_matrix.nrows), indices))  # type: ignore[attr-defined]
+
+    @staticmethod
+    def _get_mask_diff(old_mask: np.ndarray, new_mask: np.ndarray) -> List[int]:
+        return np.argwhere(old_mask != new_mask).ravel().tolist()

scaler/utility/identifiers.py

@@ -0,0 +1,107 @@
+import abc
+import hashlib
+import uuid
+from typing import Optional
+
+
+class Identifier(bytes, metaclass=abc.ABCMeta):
+    @abc.abstractmethod
+    def __repr__(self) -> str:
+        raise NotImplementedError()
+
+
+class ClientID(Identifier):
+    def __repr__(self) -> str:
+        return f"ClientID({self.decode()})"
+
+    @staticmethod
+    def generate_client_id(name: Optional[str] = None) -> "ClientID":
+        unique_client_tag = uuid.uuid4().bytes.hex()
+
+        if name is None:
+            return ClientID(f"Client|{unique_client_tag}".encode())
+        else:
+            return ClientID(f"Client|{name}|{unique_client_tag}".encode())
+
+
+class WorkerID(Identifier):
+    def __repr__(self) -> str:
+        return f"WorkerID({self.decode()})"
+
+    def is_valid(self) -> bool:
+        return self != _INVALID_WORKER_ID
+
+    @staticmethod
+    def invalid_worker_id() -> "WorkerID":
+        return _INVALID_WORKER_ID
+
+    @staticmethod
+    def generate_worker_id(name: str) -> "WorkerID":
+        unique_worker_tag = uuid.uuid4().bytes.hex()
+        return WorkerID(f"Worker|{name}|{unique_worker_tag}".encode())
+
+
+_INVALID_WORKER_ID = WorkerID(b"")
+
+
+class ProcessorID(Identifier):
+    def __repr__(self) -> str:
+        return f"ProcessorID({self.hex()})"
+
+    @staticmethod
+    def generate_processor_id() -> "ProcessorID":
+        return ProcessorID(uuid.uuid4().bytes)
+
+
+class TaskID(Identifier):
+    def __repr__(self) -> str:
+        return f"TaskID({self.hex()})"
+
+    @staticmethod
+    def generate_task_id() -> "TaskID":
+        return TaskID(uuid.uuid4().bytes)
+
+
+class ObjectID(bytes):
+    SERIALIZER_TAG = hashlib.md5(b"serializer").digest()
+
+    """
+    Scaler 32-bytes object IDs.
+
+    Object ID are built from 2x16-bytes parts:
+
+    - the first 16-bytes uniquely identify the owner of the object (i.e. the Scaler client's hash);
+    - the second 16-bytes uniquely identify the object's content.
+    """
+
+    def __new__(cls, value: bytes):
+        if len(value) != 32:
+            raise ValueError("Scaler object ID must be 32 bytes.")
+
+        return super().__new__(cls, value)
+
+    @staticmethod
+    def generate_object_id(owner: ClientID) -> "ObjectID":
+        owner_hash = hashlib.md5(owner).digest()
+        unique_object_tag = uuid.uuid4().bytes
+        return ObjectID(owner_hash + unique_object_tag)
+
+    @staticmethod
+    def generate_serializer_object_id(owner: ClientID) -> "ObjectID":
+        owner_hash = hashlib.md5(owner).digest()
+        return ObjectID(owner_hash + ObjectID.SERIALIZER_TAG)
+
+    def owner_hash(self) -> bytes:
+        return self[:16]
+
+    def object_tag(self) -> bytes:
+        return self[16:]
+
+    def is_serializer(self) -> bool:
+        return self.object_tag() == ObjectID.SERIALIZER_TAG
+
+    def is_owner(self, owner: ClientID) -> bool:
+        return hashlib.md5(owner).digest() == self.owner_hash()
+
+    def __repr__(self) -> str:
+        return f"ObjectID(owner_hash={self.owner_hash().hex()}, object_tag={self.object_tag().hex()})"

scaler/utility/logging/decorators.py

@@ -0,0 +1,25 @@
+import functools
+import inspect
+import logging
+import typing
+
+from scaler.utility.logging.scoped_logger import ScopedLogger
+
+
+def log_function(level_number: int = 2, logging_level: int = logging.INFO) -> typing.Callable:
+    def decorator(func: typing.Callable) -> typing.Callable:
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            with ScopedLogger(
+                f"execute {func.__name__} at {get_caller_location(level_number)}", logging_level=logging_level
+            ):
+                return func(*args, **kwargs)
+
+        return wrapper
+
+    return decorator
+
+
+def get_caller_location(stack_level: int):
+    caller = inspect.getframeinfo(inspect.stack()[stack_level][0])
+    return f"{caller.filename}:{caller.lineno}"

scaler/utility/logging/scoped_logger.py

@@ -0,0 +1,33 @@
+import datetime
+import logging
+import time
+from typing import Optional
+
+
+class ScopedLogger:
+    def __init__(self, message: str, logging_level=logging.INFO):
+        self.timer = TimedLogger(message=message, logging_level=logging_level)
+
+    def __enter__(self):
+        self.timer.begin()
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.timer.end()
+
+
+class TimedLogger:
+    def __init__(self, message: str, logging_level=logging.INFO):
+        self.message = message
+        self.logging_level = logging_level
+        self.timer: Optional[int] = None
+
+    def begin(self):
+        self.timer = time.perf_counter_ns()
+        logging.log(self.logging_level, f"beginning {self.message}")
+
+    def end(self):
+        elapsed = time.perf_counter_ns() - self.timer
+        offset = datetime.timedelta(
+            seconds=int(elapsed / 1e9), milliseconds=int(elapsed % 1e9 / 1e6), microseconds=int(elapsed % 1e6 / 1e3)
+        )
+        logging.log(self.logging_level, f"completed {self.message} in {offset}")