opengris-scaler 1.12.7__cp311-cp311-manylinux_2_28_x86_64.whl

scaler/io/sync_subscriber.py

@@ -0,0 +1,83 @@
+import logging
+import threading
+from typing import Callable, Optional
+
+import zmq
+
+from scaler.io.mixins import SyncSubscriber
+from scaler.io.utility import deserialize
+from scaler.protocol.python.mixins import Message
+from scaler.config.types.zmq import ZMQConfig
+
+
+class ZMQSyncSubscriber(SyncSubscriber, threading.Thread):
+    def __init__(
+        self,
+        address: ZMQConfig,
+        callback: Callable[[Message], None],
+        topic: bytes,
+        exit_callback: Optional[Callable[[], None]] = None,
+        stop_event: threading.Event = threading.Event(),
+        daemonic: bool = False,
+        timeout_seconds: int = -1,
+    ):
+        threading.Thread.__init__(self)
+
+        self._stop_event = stop_event
+        self._address = address
+        self._callback = callback
+        self._exit_callback = exit_callback
+        self._topic = topic
+        self.daemon = bool(daemonic)
+        self._timeout_seconds = timeout_seconds
+
+        self._context: Optional[zmq.Context] = None
+        self._socket: Optional[zmq.Socket] = None
+
+    def __close(self):
+        self._socket.close()
+
+    def __stop_polling(self):
+        self._stop_event.set()
+
+    def destroy(self):
+        self.__stop_polling()
+
+    def run(self) -> None:
+        self.__initialize()
+
+        while not self._stop_event.is_set():
+            self.__routine_polling()
+
+        if self._exit_callback is not None:
+            self._exit_callback()
+
+        self.__close()
+
+    def __initialize(self):
+        self._context = zmq.Context.instance()
+        self._socket = self._context.socket(zmq.SUB)
+        self._socket.setsockopt(zmq.RCVHWM, 0)
+
+        if self._timeout_seconds == -1:
+            self._socket.setsockopt(zmq.RCVTIMEO, self._timeout_seconds)
+        else:
+            self._socket.setsockopt(zmq.RCVTIMEO, self._timeout_seconds * 1000)
+
+        self._socket.subscribe(self._topic)
+        self._socket.connect(self._address.to_address())
+        self._socket.connect(self._address.to_address())
+
+    def __routine_polling(self):
+        try:
+            self.__routine_receive(self._socket.recv(copy=False).bytes)
+        except zmq.Again:
+            raise TimeoutError(f"Cannot connect to {self._address.to_address()} in {self._timeout_seconds} seconds")
+
+    def __routine_receive(self, payload: bytes):
+        result: Optional[Message] = deserialize(payload)
+        if result is None:
+            logging.error(f"received unknown message: {payload!r}")
+            return None
+
+        self._callback(result)

scaler/io/utility.py

@@ -0,0 +1,31 @@
+import logging
+from typing import List, Optional
+
+from scaler.config.defaults import CAPNP_DATA_SIZE_LIMIT, CAPNP_MESSAGE_SIZE_LIMIT
+from scaler.protocol.capnp._python import _message  # noqa
+from scaler.protocol.python.message import PROTOCOL
+from scaler.protocol.python.mixins import Message
+
+
+def deserialize(data: bytes) -> Optional[Message]:
+    with _message.Message.from_bytes(data, traversal_limit_in_words=CAPNP_MESSAGE_SIZE_LIMIT) as payload:
+        if not hasattr(payload, payload.which()):
+            logging.error(f"unknown message type: {payload.which()}")
+            return None
+
+        message = getattr(payload, payload.which())
+        return PROTOCOL[payload.which()](message)
+
+
+def serialize(message: Message) -> bytes:
+    payload = _message.Message(**{PROTOCOL.inverse[type(message)]: message.get_message()})
+    return payload.to_bytes()
+
+
+def chunk_to_list_of_bytes(data: bytes) -> List[bytes]:
+    # TODO: change to list of memoryview when capnp can support memoryview
+    return [data[i : i + CAPNP_DATA_SIZE_LIMIT] for i in range(0, len(data), CAPNP_DATA_SIZE_LIMIT)]
+
+
+def concat_list_of_bytes(data: List[bytes]) -> bytes:
+    return bytearray().join(data)

scaler/io/ymq/CMakeLists.txt

@@ -0,0 +1,98 @@
+add_library(ymq_objs OBJECT
+    bytes.h
+    common.h
+    configuration.h
+
+    epoll_context.h
+    epoll_context.cpp
+
+    iocp_context.h
+    iocp_context.cpp
+
+    event_loop.h
+
+    event_loop_thread.h
+    event_loop_thread.cpp
+
+    event_manager.h
+    # file_descriptor.h
+
+    message_connection.h
+    message_connection_tcp.h
+    message_connection_tcp.cpp
+
+    third_party/concurrentqueue.h
+    interruptive_concurrent_queue.h
+
+    typedefs.h
+
+    io_context.h
+    io_context.cpp
+
+    io_socket.h
+    io_socket.cpp
+
+    tcp_server.h
+    tcp_server.cpp
+
+    tcp_client.h
+    tcp_client.cpp
+
+    tcp_operations.h
+
+    timestamp.h
+
+    timed_queue.h
+
+    network_utils.h
+
+    error.h
+
+    logging.h
+
+    simple_interface.h
+    simple_interface.cpp
+)
+
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/scaler/io/ymq)
+
+if(LINUX)
+    # ymq python =======================================================================================================
+    find_package(Python3 COMPONENTS Development.Module REQUIRED)
+
+    add_library(py_ymq SHARED
+        pymod_ymq/bytes.h
+        pymod_ymq/exception.h
+        pymod_ymq/gil.h
+        pymod_ymq/message.h
+        pymod_ymq/io_context.h
+        pymod_ymq/io_socket.h
+        pymod_ymq/ymq.h
+        pymod_ymq/ymq.cpp
+    )
+
+    # target_include_directories(py_ymq PRIVATE ${Python3_INCLUDE_DIRS})
+    target_link_libraries(py_ymq PRIVATE
+        $<$<CXX_COMPILER_ID:GNU,Clang>:-Wl,-Bstatic> ymq_objs
+        $<$<CXX_COMPILER_ID:GNU,Clang>:-Wl,-Bdynamic>
+        Python3::Module
+    )
+
+    set_target_properties(py_ymq PROPERTIES
+        PREFIX ""
+        OUTPUT_NAME "_ymq"
+        LINKER_LANGUAGE CXX
+    )
+
+    install(
+        TARGETS py_ymq
+        LIBRARY DESTINATION scaler/io/ymq
+    )
+endif()
+
+if(WIN32)
+    # ymq python =======================================================================================================
+
+    target_link_libraries(ymq_objs PRIVATE "ws2_32")
+    target_compile_definitions(ymq_objs PRIVATE _WINSOCKAPI_=) # Yes, trailing equal to guarantee empty def
+endif()

scaler/io/ymq/_ymq.pyi

@@ -0,0 +1,96 @@
+# NOTE: NOT IMPLEMENTATION, TYPE INFORMATION ONLY
+# This file contains type stubs for the Ymq Python C Extension module
+import sys
+from enum import IntEnum
+from typing import Callable, Optional, SupportsBytes, Union
+
+if sys.version_info >= (3, 12):
+    from collections.abc import Buffer
+else:
+    Buffer = object
+
+class Bytes(Buffer):
+    data: bytes | None
+    len: int
+
+    def __init__(self, data: Buffer | None = None) -> None: ...
+    def __repr__(self) -> str: ...
+    def __len__(self) -> int: ...
+
+    # this type signature is not 100% accurate because it's implemented in C
+    # but this satisfies the type check and is good enough
+    def __buffer__(self, flags: int, /) -> memoryview: ...
+
+class Message:
+    address: Bytes | None
+    payload: Bytes
+
+    def __init__(
+        self, address: Bytes | bytes | SupportsBytes | None, payload: Bytes | bytes | SupportsBytes
+    ) -> None: ...
+    def __repr__(self) -> str: ...
+    def __str__(self) -> str: ...
+
+class IOSocketType(IntEnum):
+    Uninit = 0
+    Binder = 1
+    Connector = 2
+    Unicast = 3
+    Multicast = 4
+
+class BaseIOContext:
+    num_threads: int
+
+    def __init__(self, num_threads: int = 1) -> None: ...
+    def __repr__(self) -> str: ...
+    def createIOSocket(
+        self, callback: Callable[[Union[BaseIOSocket, Exception]], None], identity: str, socket_type: IOSocketType
+    ) -> None:
+        """Create an io socket with an identity and socket type"""
+
+class BaseIOSocket:
+    identity: str
+    socket_type: IOSocketType
+
+    def __repr__(self) -> str: ...
+    def send(self, callback: Callable[[Optional[Exception]], None], message: Message) -> None:
+        """Send a message to one of the socket's peers"""
+
+    def recv(self, callback: Callable[[Union[Message, Exception]], None]) -> None:
+        """Receive a message from one of the socket's peers"""
+
+    def bind(self, callback: Callable[[Optional[Exception]], None], address: str) -> None:
+        """Bind the socket to an address and listen for incoming connections"""
+
+    def connect(self, callback: Callable[[Optional[Exception]], None], address: str) -> None:
+        """Connect to a remote socket"""
+
+class ErrorCode(IntEnum):
+    Uninit = 0
+    InvalidPortFormat = 1
+    InvalidAddressFormat = 2
+    ConfigurationError = 3
+    SignalNotSupported = 4
+    CoreBug = 5
+    RepetetiveIOSocketIdentity = 6
+    RedundantIOSocketRefCount = 7
+    MultipleConnectToNotSupported = 8
+    MultipleBindToNotSupported = 9
+    InitialConnectFailedWithInProgress = 10
+    SendMessageRequestCouldNotComplete = 11
+    SetSockOptNonFatalFailure = 12
+    IPv6NotSupported = 13
+    RemoteEndDisconnectedOnSocketWithoutGuaranteedDelivery = 14
+    ConnectorSocketClosedByRemoteEnd = 15
+    IOSocketStopRequested = 16
+    BinderSendMessageWithNoAddress = 17
+
+    def explanation(self) -> str: ...
+
+class YMQException(Exception):
+    code: ErrorCode
+    message: str
+
+    def __init__(self, /, code: ErrorCode, message: str) -> None: ...
+    def __repr__(self) -> str: ...
+    def __str__(self) -> str: ...

scaler/io/ymq/bytes.h

@@ -0,0 +1,114 @@
+#pragma once
+
+// C
+#include <string.h>  // memcmp
+
+#include <algorithm>
+#include <compare>
+#include <cstddef>
+#include <cstdint>
+#include <cstring>
+
+// C++
+#include <optional>
+#include <string>
+
+// First-party
+#include "scaler/io/ymq/common.h"
+
+// TODO: This is not in the namespace
+class Bytes {
+    uint8_t* _data;
+    size_t _len;
+
+    void free()
+    {
+        if (is_null())
+            return;
+        delete[] _data;
+        _data = nullptr;
+    }
+
+    explicit Bytes(uint8_t* data, size_t len): _data(data), _len(len) {}
+
+public:
+    Bytes(char* data, size_t len): _data(datadup((uint8_t*)data, len)), _len(len) {}
+
+    explicit Bytes(const std::string& s): _data(datadup((uint8_t*)s.data(), s.length())), _len(s.length()) {}
+
+    Bytes(): _data {}, _len {} {}
+
+    Bytes(const Bytes& other) noexcept
+    {
+        this->_data = datadup(other._data, other._len);
+        this->_len  = other._len;
+    }
+
+    Bytes& operator=(const Bytes& other) noexcept
+    {
+        Bytes tmp(other);
+        swap(*this, tmp);
+        return *this;
+    }
+
+    friend void swap(Bytes& x, Bytes& y) noexcept
+    {
+        using std::swap;
+        swap(x._len, y._len);
+        swap(x._data, y._data);
+    }
+
+    Bytes(Bytes&& other) noexcept: _data(other._data), _len(other._len)
+    {
+        other._data = nullptr;
+        other._len  = 0;
+    }
+
+    friend std::strong_ordering operator<=>(const Bytes& x, const Bytes& y) noexcept
+    {
+        return std::lexicographical_compare_three_way(x._data, x._data + x._len, y._data, y._data + y._len);
+    }
+
+    // https://stackoverflow.com/questions/68221024/why-must-i-provide-operator-when-operator-is-enough
+    friend bool operator==(const Bytes& x, const Bytes& y) noexcept { return x <=> y == 0; }
+
+    Bytes& operator=(Bytes&& other) noexcept
+    {
+        if (this != &other) {
+            this->free();  // free current data
+
+            _data = other._data;
+            _len  = other._len;
+
+            other._data = nullptr;
+            other._len  = 0;
+        }
+        return *this;
+    }
+
+    ~Bytes() { this->free(); }
+
+    [[nodiscard]] constexpr bool operator!() const noexcept { return is_null(); }
+
+    [[nodiscard]] constexpr bool is_null() const noexcept { return !this->_data; }
+
+    std::optional<std::string> as_string() const
+    {
+        if (is_null())
+            return std::nullopt;
+
+        return std::string((char*)_data, _len);
+    }
+
+    [[nodiscard("Allocated Bytes is not used, likely causing a memory leak")]]
+    static Bytes alloc(size_t len) noexcept
+    {
+        auto ptr = new uint8_t[len];  // we just assume the allocation will succeed
+        return Bytes {ptr, len};
+    }
+
+    [[nodiscard]] constexpr size_t len() const { return _len; }
+    [[nodiscard]] constexpr size_t size() const { return _len; }
+    [[nodiscard]] constexpr const uint8_t* data() const { return _data; }
+    [[nodiscard]] constexpr uint8_t* data() { return _data; }
+};

scaler/io/ymq/common.h

@@ -0,0 +1,29 @@
+#pragma once
+
+// C++
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+
+using Errno = int;
+
+[[nodiscard("Memory is allocated but not used, likely causing a memory leak")]]
+inline uint8_t* datadup(const uint8_t* data, size_t len) noexcept
+{
+    uint8_t* dup = new uint8_t[len];  // we just assume allocation will succeed
+    std::memcpy(dup, data, len);
+    return dup;
+}
+
+inline void serialize_u32(uint32_t x, uint8_t buffer[4])
+{
+    buffer[0] = x & 0xFF;
+    buffer[1] = (x >> 8) & 0xFF;
+    buffer[2] = (x >> 16) & 0xFF;
+    buffer[3] = (x >> 24) & 0xFF;
+}
+
+inline void deserialize_u32(const uint8_t buffer[4], uint32_t* x)
+{
+    *x = buffer[0] | buffer[1] << 8 | buffer[2] << 16 | buffer[3] << 24;
+}

scaler/io/ymq/configuration.h

@@ -0,0 +1,60 @@
+#pragma once
+
+// C++
+#include <expected>
+#include <functional>
+#include <memory>
+#include <string>
+
+// Because the devil says "You shall live with errors".
+// ^-- The linker complains when the file is not here.
+#include "scaler/io/ymq/error.h"
+
+// Because the devil casts spells in plain English.
+#ifdef _WIN32
+#undef SendMessageCallback
+#define __PRETTY_FUNCTION__ __FUNCSIG__
+#endif  // _WIN32
+
+namespace scaler {
+namespace ymq {
+
+class EpollContext;
+class IocpContext;
+class Message;
+class IOSocket;
+
+// Use feature-test macro to detect support for std::move_only_function.
+// This works across GCC, Clang, and MSVC on all platforms.
+#if defined(__cpp_lib_move_only_function) && __cpp_lib_move_only_function >= 202110L
+template <typename T>
+using MoveOnlyFunction = std::move_only_function<T>;
+#else
+template <typename T>
+using MoveOnlyFunction = std::function<T>;
+#endif
+
+constexpr const uint64_t IOCP_SOCKET_CLOSED   = 4;
+constexpr const uint64_t LARGEST_PAYLOAD_SIZE = 6000'000'000'000;  // 6TB
+
+struct Configuration {
+#ifdef __linux__
+    using PollingContext = EpollContext;
+#endif  // __linux__
+#ifdef _WIN32
+    using PollingContext = IocpContext;
+#endif  // _WIN32
+
+    using IOSocketIdentity                = std::string;
+    using SendMessageCallback             = MoveOnlyFunction<void(std::expected<void, Error>)>;
+    using RecvMessageCallback             = MoveOnlyFunction<void(std::pair<Message, Error>)>;
+    using ConnectReturnCallback           = MoveOnlyFunction<void(std::expected<void, Error>)>;
+    using BindReturnCallback              = MoveOnlyFunction<void(std::expected<void, Error>)>;
+    using CreateIOSocketCallback          = MoveOnlyFunction<void(std::shared_ptr<IOSocket>)>;
+    using TimedQueueCallback              = MoveOnlyFunction<void()>;
+    using ExecutionFunction               = MoveOnlyFunction<void()>;
+    using ExecutionCancellationIdentifier = size_t;
+};
+
+}  // namespace ymq
+}  // namespace scaler

scaler/io/ymq/epoll_context.cpp

@@ -0,0 +1,185 @@
+#ifdef __linux__
+
+#include "scaler/io/ymq/epoll_context.h"
+
+#include <sys/epoll.h>
+
+#include <cerrno>
+#include <functional>
+
+#include "scaler/io/ymq/error.h"
+#include "scaler/io/ymq/event_manager.h"
+
+namespace scaler {
+namespace ymq {
+
+void EpollContext::execPendingFunctions()
+{
+    while (_delayedFunctions.size()) {
+        auto top = std::move(_delayedFunctions.front());
+        top();
+        _delayedFunctions.pop();
+    }
+}
+
+void EpollContext::loop()
+{
+    std::array<epoll_event, _reventSize> events {};
+    int n = epoll_wait(_epfd, events.data(), _reventSize, -1);
+    if (n == -1) {
+        const int myErrno = errno;
+        switch (myErrno) {
+            case EINTR:
+                // unrecoverableError({
+                //     Error::ErrorCode::SignalNotSupported,
+                //     "Originated from",
+                //     "epoll_wait(2)",
+                //     "Errno is",
+                //     strerror(errno),
+                // });
+
+                // todo: investigate better error handling
+                // the epoll thread is not expected to receive signals(?)
+                // but occasionally does (e.g. sigwinch) and we shouldn't stop the thread in that case
+                break;
+            case EBADF:
+            case EFAULT:
+            case EINVAL:
+            default:
+                unrecoverableError({
+                    Error::ErrorCode::CoreBug,
+                    "Originated from",
+                    "epoll_wait(2)",
+                    "Errno is",
+                    strerror(errno),
+                    "_epfd",
+                    _epfd,
+                    "_reventSize",
+                    _reventSize,
+                });
+                break;
+        }
+    }
+
+    for (auto it = events.begin(); it != events.begin() + n; ++it) {
+        epoll_event current_event = *it;
+        auto* event               = (EventManager*)current_event.data.ptr;
+        if (event == (void*)_isInterruptiveFd) {
+            auto vec = _interruptiveFunctions.dequeue();
+            std::ranges::for_each(vec, [](auto&& x) { x(); });
+        } else if (event == (void*)_isTimingFd) {
+            auto vec = _timingFunctions.dequeue();
+            std::ranges::for_each(vec, [](auto& x) { x(); });
+        } else {
+            event->onEvents(current_event.events);
+        }
+    }
+
+    execPendingFunctions();
+}
+
+void EpollContext::addFdToLoop(int fd, uint64_t events, EventManager* manager)
+{
+    epoll_event event {};
+    event.events   = (int)events & (EPOLLIN | EPOLLOUT | EPOLLET);
+    event.data.ptr = (void*)manager;
+    int res        = epoll_ctl(_epfd, EPOLL_CTL_ADD, fd, &event);
+    if (res == 0) {
+        return;
+    }
+
+    if (errno == EEXIST) {
+        if (epoll_ctl(_epfd, EPOLL_CTL_MOD, fd, &event) == 0) {
+            return;
+        }
+    }
+
+    const int myErrno = errno;
+    switch (myErrno) {
+        case ENOMEM:
+        case ENOSPC:
+            unrecoverableError({
+                Error::ErrorCode::ConfigurationError,
+                "Originated from",
+                "epoll_ctl(2)",
+                "Errno is",
+                strerror(errno),
+                "_epfd",
+                _epfd,
+                "fd",
+                fd,
+                "event.events",
+                (int)event.events,
+            });
+            break;
+
+        case EBADF:
+        case EPERM:
+        case EINVAL:
+        case ELOOP:
+        case ENOENT:
+        default:
+            unrecoverableError({
+                Error::ErrorCode::CoreBug,
+                "Originated from",
+                "epoll_ctl(2)",
+                "Errno is",
+                strerror(errno),
+                "_epfd",
+                _epfd,
+                "fd",
+                fd,
+                "event.events",
+                (int)event.events,
+            });
+            break;
+    }
+}
+
+void EpollContext::removeFdFromLoop(int fd)
+{
+    if (epoll_ctl(_epfd, EPOLL_CTL_DEL, fd, nullptr) == 0) {
+        return;
+    }
+    const int myErrno = errno;
+    switch (myErrno) {
+        case ENOMEM:
+        case ENOSPC:
+            unrecoverableError({
+                Error::ErrorCode::ConfigurationError,
+                "Originated from",
+                __PRETTY_FUNCTION__,
+                "Errno is",
+                strerror(errno),
+                "_epfd",
+                _epfd,
+                "fd",
+                fd,
+            });
+            break;
+
+        case EBADF:
+        case EPERM:
+        case EINVAL:
+        case ELOOP:
+        case ENOENT:
+        default:
+            unrecoverableError({
+                Error::ErrorCode::CoreBug,
+                "Originated from",
+                __PRETTY_FUNCTION__,
+                "Errno is",
+                strerror(errno),
+                "_epfd",
+                _epfd,
+                "fd",
+                fd,
+            });
+            break;
+    }
+}
+
+}  // namespace ymq
+}  // namespace scaler
+
+#endif  // __linux__