opengris-scaler 1.12.7__cp310-cp310-musllinux_1_2_x86_64.whl

scaler/io/ymq/timed_queue.h

@@ -0,0 +1,272 @@
+#pragma once
+
+#ifdef __linux__
+#include <sys/timerfd.h>
+#include <unistd.h>
+#endif  // __linux__
+#ifdef _WIN32
+#include <windows.h>
+#endif  // _WIN32
+
+#include <cassert>
+#include <queue>
+#include <set>
+
+#include "scaler/io/ymq/configuration.h"
+#include "scaler/io/ymq/error.h"
+#include "scaler/io/ymq/timestamp.h"
+
+namespace scaler {
+namespace ymq {
+
+#ifdef __linux__
+inline int createTimerfd()
+{
+    int timerfd = ::timerfd_create(CLOCK_MONOTONIC, TFD_NONBLOCK | TFD_CLOEXEC);
+    if (timerfd >= 0) {
+        return timerfd;
+    }
+
+    const int myErrno = errno;
+    switch (myErrno) {
+        case EMFILE:
+        case ENFILE:
+        case ENODEV:
+        case ENOMEM:
+        case EPERM:
+            unrecoverableError({
+                Error::ErrorCode::ConfigurationError,
+                "Originated from",
+                __PRETTY_FUNCTION__,
+                "Errno is",
+                strerror(myErrno),
+            });
+            break;
+
+        case EINVAL:
+        case EBADF:
+        case EFAULT:
+        case ECANCELED:
+        default:
+            unrecoverableError({
+                Error::ErrorCode::CoreBug,
+                "Originated from",
+                __PRETTY_FUNCTION__,
+                "Errno is",
+                strerror(myErrno),
+            });
+            break;
+    }
+}
+
+class TimedQueue {
+public:
+    using Callback            = Configuration::TimedQueueCallback;
+    using Identifier          = Configuration::ExecutionCancellationIdentifier;
+    using TimedFunc           = std::tuple<Timestamp, Callback, Identifier>;
+    constexpr static auto cmp = [](const auto& x, const auto& y) { return std::get<0>(x) < std::get<0>(y); };
+    using PriorityQueue       = std::priority_queue<TimedFunc, std::vector<TimedFunc>, decltype(cmp)>;
+
+    TimedQueue(): _timerFd(createTimerfd()), _currentId {} { assert(_timerFd); }
+    ~TimedQueue()
+    {
+        if (_timerFd >= 0)
+            close(_timerFd);
+    }
+
+    Identifier push(Timestamp timestamp, Callback cb)
+    {
+        auto ts = convertToItimerspec(timestamp);
+        if (pq.empty() || timestamp < std::get<0>(pq.top())) {
+            int ret = timerfd_settime(_timerFd, 0, &ts, nullptr);
+            if (ret == -1) {
+                unrecoverableError({
+                    Error::ErrorCode::CoreBug,
+                    "Originated from",
+                    __PRETTY_FUNCTION__,
+                    "Errno is",
+                    strerror(errno),
+                });
+            }
+        }
+        pq.push({timestamp, std::move(cb), _currentId});
+        return _currentId++;
+    }
+
+    void cancelExecution(Identifier id) { _cancelledFunctions.insert(id); }
+
+    std::vector<Callback> dequeue()
+    {
+        uint64_t numItems;
+        ssize_t n = read(_timerFd, &numItems, sizeof numItems);
+        if (n != sizeof numItems) [[unlikely]] {
+            // This should never happen anyway
+            unrecoverableError({
+                Error::ErrorCode::CoreBug,
+                "Originated from",
+                __PRETTY_FUNCTION__,
+                "Errno is",
+                strerror(errno),
+            });
+        }
+
+        std::vector<Callback> callbacks;
+
+        Timestamp now;
+        while (pq.size()) {
+            if (std::get<0>(pq.top()) < now) {
+                auto [ts, cb, id] = std::move(const_cast<PriorityQueue::reference>(pq.top()));
+                pq.pop();
+                auto cancelled = _cancelledFunctions.find(id);
+                if (cancelled != _cancelledFunctions.end()) {
+                    _cancelledFunctions.erase(cancelled);
+                } else {
+                    callbacks.emplace_back(std::move(cb));
+                }
+            } else
+                break;
+        }
+
+        if (!pq.empty()) {
+            auto nextTs = std::get<0>(pq.top());
+            auto ts     = convertToItimerspec(nextTs);
+            int ret     = timerfd_settime(_timerFd, 0, &ts, nullptr);
+            if (ret == -1) {
+                const int myErrno = errno;
+                switch (myErrno) {
+                    case EMFILE:
+                    case ENFILE:
+                    case ENODEV:
+                    case ENOMEM:
+                    case EPERM:
+                        unrecoverableError({
+                            Error::ErrorCode::ConfigurationError,
+                            "Originated from",
+                            __PRETTY_FUNCTION__,
+                            "Errno is",
+                            strerror(myErrno),
+                        });
+                        break;
+
+                    case EINVAL:
+                    case EBADF:
+                    case EFAULT:
+                    case ECANCELED:
+                    default:
+                        unrecoverableError({
+                            Error::ErrorCode::CoreBug,
+                            "Originated from",
+                            __PRETTY_FUNCTION__,
+                            "Errno is",
+                            strerror(myErrno),
+                        });
+                        break;
+                }
+            }
+        }
+        return callbacks;
+    }
+
+    int timingFd() const { return _timerFd; }
+
+private:
+    int _timerFd;
+    Identifier _currentId;
+    PriorityQueue pq;
+    std::set<Identifier> _cancelledFunctions;
+};
+
+#endif  // __linux__
+
+#ifdef _WIN32
+class TimedQueue {
+public:
+    using Callback            = Configuration::TimedQueueCallback;
+    using Identifier          = Configuration::ExecutionCancellationIdentifier;
+    using TimedFunc           = std::tuple<Timestamp, Callback, Identifier>;
+    constexpr static auto cmp = [](const auto& x, const auto& y) { return std::get<0>(x) < std::get<0>(y); };
+    using PriorityQueue       = std::priority_queue<TimedFunc, std::vector<TimedFunc>, decltype(cmp)>;
+    HANDLE _completionPort;
+    const size_t _key;
+
+    // TODO: Handle error for system calls
+    TimedQueue(HANDLE completionPort, size_t key)
+        : _completionPort(completionPort), _key(key), _timerFd(CreateWaitableTimer(NULL, 0, NULL)), _currentId {}
+    {
+        assert(_timerFd);
+    }
+    ~TimedQueue()
+    {
+        if (_timerFd) {
+            CloseHandle(_timerFd);
+        }
+    }
+
+    Identifier push(Timestamp timestamp, Callback cb)
+    {
+        auto ts = convertToLARGE_INTEGER(timestamp);
+        if (pq.empty() || timestamp < std::get<0>(pq.top())) {
+            SetWaitableTimer(
+                _timerFd,
+                (LARGE_INTEGER*)&ts,
+                0,
+                [](LPVOID thisPointer, DWORD, DWORD) {
+                    auto* self = (TimedQueue*)thisPointer;
+                    PostQueuedCompletionStatus(self->_completionPort, 0, (ULONG_PTR)(self->_timerFd), nullptr);
+                },
+                (LPVOID)this,
+                false);
+        }
+        pq.push({timestamp, std::move(cb), _currentId});
+        return _currentId++;
+    }
+
+    void cancelExecution(Identifier id) { _cancelledFunctions.insert(id); }
+
+    std::vector<Callback> dequeue()
+    {
+        std::vector<Callback> callbacks;
+
+        Timestamp now;
+        while (pq.size()) {
+            if (std::get<0>(pq.top()) < now) {
+                auto [ts, cb, id] = std::move(const_cast<PriorityQueue::reference>(pq.top()));
+                pq.pop();
+                auto cancelled = _cancelledFunctions.find(id);
+                if (cancelled != _cancelledFunctions.end()) {
+                    _cancelledFunctions.erase(cancelled);
+                } else {
+                    callbacks.emplace_back(std::move(cb));
+                }
+            } else
+                break;
+        }
+
+        if (!pq.empty()) {
+            auto nextTs = std::get<0>(pq.top());
+            auto ts     = convertToLARGE_INTEGER(nextTs);
+            SetWaitableTimer(
+                _timerFd,
+                (LARGE_INTEGER*)&ts,
+                0,
+                [](LPVOID thisPointer, DWORD, DWORD) {
+                    auto* self = (TimedQueue*)thisPointer;
+                    PostQueuedCompletionStatus(self->_completionPort, 0, (ULONG_PTR)(self->_timerFd), nullptr);
+                },
+                (LPVOID)this,
+                false);
+        }
+        return callbacks;
+    }
+
+private:
+    HANDLE _timerFd;
+    Identifier _currentId;
+    PriorityQueue pq;
+    std::set<Identifier> _cancelledFunctions;
+};
+
+#endif  // _WIN32
+
+}  // namespace ymq
+}  // namespace scaler

scaler/io/ymq/timestamp.h

@@ -0,0 +1,102 @@
+#pragma once
+
+#ifdef __linux__
+#include <sys/time.h>  // itimerspec
+#endif                 // __linux__
+#ifdef _WIN32
+#include <windows.h>
+#endif  // _WIN32
+
+#include <cassert>
+#include <chrono>
+#include <ostream>
+#include <sstream>  // stringify
+
+namespace scaler {
+namespace ymq {
+
+// Simple timestamp utility
+struct Timestamp {
+    std::chrono::time_point<std::chrono::system_clock> timestamp;
+
+    friend std::strong_ordering operator<=>(Timestamp x, Timestamp y) { return x.timestamp <=> y.timestamp; }
+
+    Timestamp(): timestamp(std::chrono::system_clock::now()) {}
+    Timestamp(std::chrono::time_point<std::chrono::system_clock> t) { timestamp = std::move(t); }
+
+    template <class Rep, class Period = std::ratio<1>>
+    Timestamp createTimestampByOffsetDuration(std::chrono::duration<Rep, Period> offset)
+    {
+        return {timestamp + offset};
+    }
+};
+
+// For possibly logging purposes
+inline std::string stringifyTimestamp(Timestamp ts)
+{
+    std::ostringstream oss;
+    const auto ts_point {std::chrono::floor<std::chrono::seconds>(ts.timestamp)};
+    const std::chrono::zoned_time z {std::chrono::current_zone(), ts_point};
+    oss << std::format("{0:%F %T%z}", z);
+    return oss.str();
+}
+
+inline std::ostream& operator<<(std::ostream& os, const Timestamp& ts)
+{
+    // Use the existing stringify function
+    os << stringifyTimestamp(ts);
+    return os;
+}
+
+#ifdef __linux__
+// For timerfd
+inline itimerspec convertToItimerspec(Timestamp ts)
+{
+    using namespace std::chrono;
+
+    itimerspec timerspec {};
+    const auto duration = ts.timestamp - std::chrono::system_clock::now();
+    assert(duration.count() >= 0);
+
+    const auto secs            = duration_cast<seconds>(duration);
+    const auto nanosecs        = duration_cast<nanoseconds>(duration - secs);
+    timerspec.it_value.tv_sec  = secs.count();
+    timerspec.it_value.tv_nsec = nanosecs.count();
+
+    return timerspec;
+}
+#endif  // __linux__
+#ifdef _WIN32
+// For timerfd
+inline LARGE_INTEGER convertToLARGE_INTEGER(Timestamp ts)
+{
+    using namespace std::chrono;
+    const auto duration = ts.timestamp - std::chrono::system_clock::now();
+    assert(duration.count() >= 0);
+    const auto nanosecs            = duration_cast<nanoseconds>(duration);
+    long long relativeHundredNanos = 1LL * nanosecs.count() / 100 * -1;
+    return *(LARGE_INTEGER*)&relativeHundredNanos;
+}
+#endif  // _WIN32
+
+}  // namespace ymq
+}  // namespace scaler
+
+template <>
+struct std::formatter<scaler::ymq::Timestamp, char> {
+    template <class ParseContext>
+    constexpr ParseContext::iterator parse(ParseContext& ctx)
+    {
+        return ctx.begin();
+    }
+
+    template <class FmtContext>
+    constexpr FmtContext::iterator format(scaler::ymq::Timestamp e, FmtContext& ctx) const
+    {
+        std::ostringstream out;
+        const auto ts {std::chrono::floor<std::chrono::seconds>(e.timestamp)};
+        const std::chrono::zoned_time z {std::chrono::current_zone(), ts};
+        out << std::format("{0:%F %T%z}", z);
+        return std::ranges::copy(std::move(out).str(), ctx.out()).out;
+    }
+};

scaler/io/ymq/typedefs.h

@@ -0,0 +1,20 @@
+#pragma once
+
+#include <cstdint>
+
+namespace scaler {
+namespace ymq {
+
+// THIS FILE MUST NOT CONTAIN USER DEFINED TYPES
+enum IOSocketType : uint8_t {
+    Uninit,  // Not allowed from user code
+    Binder,
+    Connector,
+    Unicast,
+    Multicast,
+};
+
+enum Ownership { Owned, Borrowed };
+
+}  // namespace ymq
+}  // namespace scaler

scaler/io/ymq/utils.h

@@ -0,0 +1,34 @@
+#pragma once
+
+#include <algorithm>
+#include <cstring>
+#include <format>
+#include <string>
+
+namespace scaler {
+namespace ymq {
+
+template <std::size_t N>
+    requires(N > 0)
+consteval auto getFormatString()
+{
+    std::string str = "{}";
+    for (size_t i = 1; i < N; ++i)
+        str += ": {}";
+    std::array<char, (N - 1) * 4 + 2> arr;
+    std::ranges::copy(str, arr.begin());
+    return arr;
+}
+
+// NOTE: It seems like this two lines cannot be placed in the constructor for unknown reason.
+template <typename... Args>
+constexpr std::string argsToString(Args&&... args)
+{
+    static constexpr const auto str = getFormatString<sizeof...(Args)>();
+
+    std::string res = std::format(std::string_view {str}, std::forward<Args>(args)...);
+    return res;
+}
+
+}  // namespace ymq
+}  // namespace scaler

scaler/io/ymq/ymq.py

@@ -0,0 +1,130 @@
+# This file wraps the interface exported by the C implementation of the module
+# and provides a more ergonomic interface supporting both asynchronous and synchronous execution
+
+__all__ = ["IOSocket", "IOContext", "Message", "IOSocketType", "YMQException", "Bytes", "ErrorCode"]
+
+import asyncio
+import concurrent.futures
+from typing import Optional, Callable, TypeVar, Union
+
+try:
+    from typing import ParamSpec, Concatenate  # type: ignore[attr-defined]
+except ImportError:
+    from typing_extensions import ParamSpec, Concatenate  # type: ignore[assignment]
+
+from scaler.io.ymq._ymq import BaseIOContext, BaseIOSocket, Bytes, ErrorCode, IOSocketType, Message, YMQException
+
+
+class IOSocket:
+    _base: BaseIOSocket
+
+    def __init__(self, base: BaseIOSocket) -> None:
+        self._base = base
+
+    @property
+    def socket_type(self) -> IOSocketType:
+        return self._base.socket_type
+
+    @property
+    def identity(self) -> str:
+        return self._base.identity
+
+    async def bind(self, address: str) -> None:
+        """Bind the socket to an address and listen for incoming connections"""
+        await call_async(self._base.bind, address)
+
+    def bind_sync(self, address: str, /, timeout: Optional[float] = None) -> None:
+        """Bind the socket to an address and listen for incoming connections"""
+        call_sync(self._base.bind, address, timeout=timeout)
+
+    async def connect(self, address: str) -> None:
+        """Connect to a remote socket"""
+        await call_async(self._base.connect, address)
+
+    def connect_sync(self, address: str, /, timeout: Optional[float] = None) -> None:
+        """Connect to a remote socket"""
+        call_sync(self._base.connect, address, timeout=timeout)
+
+    async def send(self, message: Message) -> None:
+        """Send a message to one of the socket's peers"""
+        await call_async(self._base.send, message)
+
+    def send_sync(self, message: Message, /, timeout: Optional[float] = None) -> None:
+        """Send a message to one of the socket's peers"""
+        call_sync(self._base.send, message, timeout=timeout)
+
+    async def recv(self) -> Message:
+        """Receive a message from one of the socket's peers"""
+        return await call_async(self._base.recv)
+
+    def recv_sync(self, /, timeout: Optional[float] = None) -> Message:
+        """Receive a message from one of the socket's peers"""
+        return call_sync(self._base.recv, timeout=timeout)
+
+
+class IOContext:
+    _base: BaseIOContext
+
+    def __init__(self, num_threads: int = 1) -> None:
+        self._base = BaseIOContext(num_threads)
+
+    @property
+    def num_threads(self) -> int:
+        return self._base.num_threads
+
+    async def createIOSocket(self, identity: str, socket_type: IOSocketType) -> IOSocket:
+        """Create an io socket with an identity and socket type"""
+        return IOSocket(await call_async(self._base.createIOSocket, identity, socket_type))
+
+    def createIOSocket_sync(self, identity: str, socket_type: IOSocketType) -> IOSocket:
+        """Create an io socket with an identity and socket type"""
+        return IOSocket(call_sync(self._base.createIOSocket, identity, socket_type))
+
+
+P = ParamSpec("P")
+T = TypeVar("T")
+
+
+async def call_async(
+    func: Callable[Concatenate[Callable[[Union[T, Exception]], None], P], None],  # type: ignore
+    *args: P.args,  # type: ignore
+    **kwargs: P.kwargs,  # type: ignore
+) -> T:
+    future = asyncio.get_event_loop().create_future()
+
+    def callback(result: Union[T, Exception]):
+        if future.done():
+            return
+
+        loop = future.get_loop()
+
+        if isinstance(result, Exception):
+            loop.call_soon_threadsafe(future.set_exception, result)
+        else:
+            loop.call_soon_threadsafe(future.set_result, result)
+
+    func(callback, *args, **kwargs)
+    return await future
+
+
+# about the ignore directives: mypy cannot properly handle typing extension's ParamSpec and Concatenate in python <=3.9
+# these type hints are correctly understood in Python 3.10+
+def call_sync(  # type: ignore[valid-type]
+    func: Callable[Concatenate[Callable[[Union[T, Exception]], None], P], None],  # type: ignore
+    *args: P.args,  # type: ignore
+    timeout: Optional[float] = None,
+    **kwargs: P.kwargs,  # type: ignore
+) -> T:  # type: ignore
+    future: concurrent.futures.Future = concurrent.futures.Future()
+
+    def callback(result: Union[T, Exception]):
+        if future.done():
+            return
+
+        if isinstance(result, Exception):
+            future.set_exception(result)
+        else:
+            future.set_result(result)
+
+    func(callback, *args, **kwargs)
+    return future.result(timeout)

scaler/object_storage/CMakeLists.txt

@@ -0,0 +1,50 @@
+# capnp_objs ===========================================================================================================
+# compile object storage schema capnp
+set(CAPNPC_SRC_PREFIX "${PROJECT_SOURCE_DIR}/scaler/protocol/capnp" CACHE STRING "" FORCE)
+set(CAPNPC_OUTPUT_DIR "${PROJECT_SOURCE_DIR}/scaler/protocol/capnp" CACHE STRING "" FORCE)
+file(MAKE_DIRECTORY "${CAPNPC_OUTPUT_DIR}")
+capnp_generate_cpp(OBJECT_STORAGE_CAPNP_SRCS OBJECT_STORAGE_CAPNP_HEADERS "../protocol/capnp/object_storage.capnp")
+
+add_library(capnp_objs OBJECT
+    ${OBJECT_STORAGE_CAPNP_SRCS}
+    ${OBJECT_STORAGE_CAPNP_HEADERS}
+)
+
+# object_storage_server_objs ===========================================================================================
+add_library(object_storage_server_objs OBJECT
+    io_helper.cpp
+    message.cpp
+    object_storage_server.cpp
+    object_manager.cpp
+)
+
+set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/scaler/object_storage)
+
+# object_storage_server python =========================================================================================
+find_package(Python3 REQUIRED COMPONENTS Development.Module)
+
+add_library(py_object_storage_server MODULE
+    pymod_object_storage_server.cpp
+)
+
+set_target_properties(py_object_storage_server PROPERTIES
+    PREFIX ""
+    OUTPUT_NAME "object_storage_server"
+)
+
+target_link_libraries(py_object_storage_server PRIVATE
+    $<$<CXX_COMPILER_ID:GNU,Clang>:-Wl,-Bstatic>
+    capnp_objs
+    ymq_objs
+    object_storage_server_objs
+    $<$<CXX_COMPILER_ID:GNU,Clang>:-Wl,-Bdynamic>
+    CapnProto::capnp
+    CapnProto::kj
+    Python3::Module
+)
+
+install(
+    TARGETS py_object_storage_server
+    LIBRARY DESTINATION scaler/object_storage/
+    COMPONENT object_storage_server
+)

scaler/object_storage/constants.h

@@ -0,0 +1,11 @@
+#pragma once
+
+namespace scaler {
+namespace object_storage {
+
+static constexpr size_t MEMORY_LIMIT_IN_BYTES = 6uz << 40;  // 6 TB
+static constexpr const char* DEFAULT_ADDR     = "127.0.0.1";
+static constexpr const char* DEFAULT_PORT     = "55555";
+
+};  // namespace object_storage
+};  // namespace scaler

scaler/object_storage/defs.h

@@ -0,0 +1,14 @@
+#pragma once
+
+#include <memory>  // std::shared_ptr
+
+#include "scaler/io/ymq/bytes.h"
+
+namespace scaler {
+namespace object_storage {
+
+using ObjectPayload       = Bytes;
+using SharedObjectPayload = std::shared_ptr<ObjectPayload>;
+
+};  // namespace object_storage
+};  // namespace scaler

scaler/object_storage/io_helper.cpp

@@ -0,0 +1,44 @@
+#include "io_helper.h"
+
+#include <netinet/in.h>  // sockaddr_in
+#include <sys/socket.h>  // socket(2)
+#include <unistd.h>      // close(2)
+
+#include <exception>
+#include <iostream>
+
+namespace scaler {
+namespace object_storage {
+
+int getAvailableTCPPort()
+{
+    int sockfd = socket(AF_INET, SOCK_STREAM, 0);
+    if (sockfd < 0) {
+        return -1;
+    }
+
+    sockaddr_in addr {};
+    addr.sin_family      = AF_INET;
+    addr.sin_addr.s_addr = htonl(INADDR_ANY);
+    addr.sin_port        = 0;  // Let OS choose port
+
+    if (bind(sockfd, reinterpret_cast<sockaddr*>(&addr), sizeof(addr)) < 0) {
+        close(sockfd);
+        throw std::runtime_error("bind() failed");
+    }
+
+    socklen_t len = sizeof(addr);
+    if (getsockname(sockfd, reinterpret_cast<sockaddr*>(&addr), &len) < 0) {
+        close(sockfd);
+        throw std::runtime_error("getsockname() failed");
+    }
+
+    const int port = ntohs(addr.sin_port);
+
+    close(sockfd);
+
+    return port;
+}
+
+};  // namespace object_storage
+};  // namespace scaler

scaler/object_storage/io_helper.h

@@ -0,0 +1,9 @@
+#pragma once
+
+namespace scaler {
+namespace object_storage {
+
+int getAvailableTCPPort();
+
+};  // namespace object_storage
+};  // namespace scaler