opengris-scaler 1.12.7__cp38-cp38-musllinux_1_2_x86_64.whl

scaler/object_storage/object_storage_server.cpp

@@ -0,0 +1,359 @@
+#include "scaler/object_storage/object_storage_server.h"
+
+#include <algorithm>
+#include <exception>
+#include <future>
+
+#include "scaler/io/ymq/configuration.h"
+#include "scaler/io/ymq/error.h"
+#include "scaler/io/ymq/simple_interface.h"
+#include "scaler/io/ymq/typedefs.h"
+#include "scaler/object_storage/message.h"
+
+namespace scaler {
+namespace object_storage {
+
+ObjectStorageServer::ObjectStorageServer()
+{
+    initServerReadyFds();
+}
+
+ObjectStorageServer::~ObjectStorageServer()
+{
+    closeServerReadyFds();
+}
+
+void ObjectStorageServer::run(
+    std::string name,
+    std::string port,
+    ObjectStorageServer::Identity identity,
+    std::string log_level,
+    std::string log_format,
+    std::vector<std::string> log_paths)
+{
+    _logger = scaler::ymq::Logger(log_format, std::move(log_paths), scaler::ymq::Logger::stringToLogLevel(log_level));
+
+    try {
+        // NOTE: Setup IOSocket synchronously here because it is a one-time thing.
+        const auto socketType {ymq::IOSocketType::Binder};
+        _ioSocket = ymq::syncCreateSocket(_ioContext, socketType, std::move(identity));
+        const std::string networkAddress {"tcp://" + name + ':' + port};
+        ymq::syncBindSocket(_ioSocket, std::move(networkAddress));
+
+        setServerReadyFd();
+
+        _logger.log(scaler::ymq::Logger::LoggingLevel::info, "ObjectStorageServer: started");
+
+        processRequests();
+
+        _ioContext.removeIOSocket(_ioSocket);
+    } catch (const std::exception& e) {
+        _logger.log(
+            scaler::ymq::Logger::LoggingLevel::error,
+            "ObjectStorageServer: unexpected server error, reason: ",
+            e.what());
+    }
+}
+
+void ObjectStorageServer::waitUntilReady()
+{
+    uint64_t value;
+    ssize_t ret = read(onServerReadyReader, &value, sizeof(uint64_t));
+
+    if (ret != sizeof(uint64_t)) {
+        _logger.log(
+            scaler::ymq::Logger::LoggingLevel::error,
+            "ObjectStorageServer: read from onServerReadyReader failed, errno=",
+            errno);
+        std::terminate();
+    }
+}
+
+void ObjectStorageServer::shutdown()
+{
+    _ioContext.requestIOSocketStop(_ioSocket);
+}
+
+void ObjectStorageServer::initServerReadyFds()
+{
+    int pipeFds[2] {};
+    int ret = pipe(pipeFds);
+
+    if (ret != 0) {
+        _logger.log(
+            scaler::ymq::Logger::LoggingLevel::error,
+            "ObjectStorageServer: create on server ready FDs failed, errno=",
+            errno);
+        std::terminate();
+    }
+
+    onServerReadyReader = pipeFds[0];
+    onServerReadyWriter = pipeFds[1];
+}
+
+void ObjectStorageServer::setServerReadyFd()
+{
+    uint64_t value = 1;
+    ssize_t ret    = write(onServerReadyWriter, &value, sizeof(uint64_t));
+
+    if (ret != sizeof(uint64_t)) {
+        _logger.log(
+            scaler::ymq::Logger::LoggingLevel::error,
+            "ObjectStorageServer: write to onServerReadyWriter failed, errno=",
+            errno);
+        std::terminate();
+    }
+}
+
+void ObjectStorageServer::closeServerReadyFds()
+{
+    const std::array<int, 2> fds {onServerReadyReader, onServerReadyWriter};
+
+    for (const int fd: fds) {
+        if (close(fd) != 0) {
+            _logger.log(scaler::ymq::Logger::LoggingLevel::error, "ObjectStorageServer: close failed, errno=", errno);
+            std::terminate();
+        }
+    }
+}
+
+void ObjectStorageServer::processRequests()
+{
+    using namespace std::chrono_literals;
+    Identity lastMessageIdentity;
+    std::map<Identity, std::pair<ObjectRequestHeader, Bytes>> identityToFullRequest;
+    while (true) {
+        try {
+            auto invalids = std::ranges::remove_if(_pendingSendMessageFuts, [](const auto& x) { return !x.valid(); });
+            _pendingSendMessageFuts.erase(invalids.begin(), invalids.end());
+
+            std::ranges::for_each(_pendingSendMessageFuts, [](auto& fut) {
+                if (fut.wait_for(0s) == std::future_status::ready) {
+                    auto error = fut.get();
+                    assert(!error);
+                }
+            });
+
+            auto maybeMessage = ymq::syncRecvMessage(_ioSocket);
+
+            if (!maybeMessage) {
+                auto error = maybeMessage.error();
+                if (error._errorCode == ymq::Error::ErrorCode::IOSocketStopRequested) {
+                    auto n = std::ranges::count_if(_pendingSendMessageFuts, [](auto& x) {
+                        return x.valid() && x.wait_for(0s) == std::future_status::timeout;
+                    });
+                    if (!n) {
+                        _logger.log(
+                            scaler::ymq::Logger::LoggingLevel::info,
+                            "ObjectStorageServer: stopped, number of messages leftover in the system = ",
+                            std::ranges::count_if(_pendingSendMessageFuts, [](auto& x) {
+                                return x.valid() && x.wait_for(0s) == std::future_status::timeout;
+                            }));
+                    }
+
+                    if (pendingRequests.size()) {
+                        _logger.log(
+                            scaler::ymq::Logger::LoggingLevel::info,
+                            "ObjectStorageServer: stopped, number of pending requests leftover in the system = ",
+                            pendingRequests.size());
+                        pendingRequests.clear();
+                    }
+                    return;
+                } else {
+                    throw error;
+                }
+            }
+
+            const auto identity        = *maybeMessage->address.as_string();
+            lastMessageIdentity        = identity;
+            const auto headerOrPayload = std::move(maybeMessage->payload);
+
+            auto it = identityToFullRequest.find(identity);
+            if (it == identityToFullRequest.end()) {
+                identityToFullRequest[identity].first = ObjectRequestHeader::fromBuffer(headerOrPayload);
+                const auto& requestType               = identityToFullRequest[identity].first.requestType;
+                if (requestType == ObjectRequestType::DUPLICATE_OBJECT_I_D ||
+                    requestType == ObjectRequestType::SET_OBJECT) {
+                    continue;
+                }
+            } else {
+                assert(it->second.first.payloadLength == headerOrPayload.len());
+                (it->second).second = std::move(headerOrPayload);
+            }
+
+            // NOTE: PostCondition of the previous statement: We have a full request here
+
+            auto request = std::move(identityToFullRequest[identity]);
+            identityToFullRequest.erase(identity);
+            auto client = std::make_shared<Client>(_ioSocket, identity);
+
+            switch (request.first.requestType) {
+                case ObjectRequestType::SET_OBJECT: {
+                    processSetRequest(std::move(client), std::move(request));
+                    break;
+                }
+                case ObjectRequestType::GET_OBJECT: {
+                    processGetRequest(std::move(client), request.first);
+                    break;
+                }
+                case ObjectRequestType::DELETE_OBJECT: {
+                    processDeleteRequest(client, request.first);
+                    break;
+                }
+                case ObjectRequestType::DUPLICATE_OBJECT_I_D: {
+                    processDuplicateRequest(client, request);
+                    break;
+                }
+            }
+        } catch (const kj::Exception& e) {
+            _ioSocket->closeConnection(std::move(lastMessageIdentity));
+            _logger.log(
+                scaler::ymq::Logger::LoggingLevel::error,
+                "ObjectStorageServer: Malformed capnp message. Connection closed, details: ",
+                e.getDescription().cStr());
+        } catch (const std::exception& e) {
+            _logger.log(
+                scaler::ymq::Logger::LoggingLevel::error, "ObjectStorageServer: unexpected error, reason: ", e.what());
+        }
+    }
+}
+
+void ObjectStorageServer::processSetRequest(
+    std::shared_ptr<Client> client, std::pair<ObjectRequestHeader, Bytes> request)
+{
+    const auto requestHeader = std::move(request.first);
+    auto requestPayload      = std::move(request.second);
+    if (requestHeader.payloadLength > MEMORY_LIMIT_IN_BYTES) {
+        throw std::runtime_error(
+            "payload length is larger than MEMORY_LIMIT_IN_BYTES=" + std::to_string(MEMORY_LIMIT_IN_BYTES));
+    }
+
+    if (requestHeader.payloadLength > std::numeric_limits<size_t>::max()) {
+        throw std::runtime_error("payload length is larger than SIZE_MAX=" + std::to_string(SIZE_MAX));
+    }
+
+    auto objectPtr = objectManager.setObject(requestHeader.objectID, std::move(requestPayload));
+
+    optionallySendPendingRequests(requestHeader.objectID, objectPtr);
+
+    ObjectResponseHeader responseHeader {
+        .objectID      = requestHeader.objectID,
+        .payloadLength = 0,
+        .responseID    = requestHeader.requestID,
+        .responseType  = ObjectResponseType::SET_O_K,
+    };
+
+    writeMessage(client, responseHeader, {});
+}
+
+void ObjectStorageServer::processGetRequest(std::shared_ptr<Client> client, const ObjectRequestHeader& requestHeader)
+{
+    auto objectPtr = objectManager.getObject(requestHeader.objectID);
+
+    if (objectPtr != nullptr) {
+        sendGetResponse(client, requestHeader, objectPtr);
+        return;
+    } else {
+        // We don't have the object yet. Send the response later after once we receive the SET request.
+        pendingRequests[requestHeader.objectID].emplace_back(client, requestHeader);
+    }
+}
+
+void ObjectStorageServer::processDeleteRequest(std::shared_ptr<Client> client, ObjectRequestHeader& requestHeader)
+{
+    bool success = objectManager.deleteObject(requestHeader.objectID);
+
+    ObjectResponseHeader responseHeader {
+        .objectID      = requestHeader.objectID,
+        .payloadLength = 0,
+        .responseID    = requestHeader.requestID,
+        .responseType  = success ? ObjectResponseType::DEL_O_K : ObjectResponseType::DEL_NOT_EXISTS,
+    };
+
+    writeMessage(client, responseHeader, {});
+}
+
+void ObjectStorageServer::processDuplicateRequest(
+    std::shared_ptr<Client> client, std::pair<ObjectRequestHeader, Bytes> request)
+{
+    auto requestHeader = std::move(request.first);
+    auto payload       = std::move(request.second);
+    if (requestHeader.payloadLength != ObjectID::bufferSize()) {
+        throw std::runtime_error(
+            "payload length should be size_of(ObjectID)=" + std::to_string(ObjectID::bufferSize()));
+    }
+
+    ObjectID originalObjectID = ObjectID::fromBuffer(payload);
+
+    auto objectPtr = objectManager.duplicateObject(originalObjectID, requestHeader.objectID);
+
+    std::vector<ObjectStorageServer::SendMessageFuture> res;
+    if (objectPtr != nullptr) {
+        optionallySendPendingRequests(requestHeader.objectID, objectPtr);
+        sendDuplicateResponse(client, requestHeader);
+    } else {
+        // We don't have the referenced original object yet. Send the response later once we receive the SET
+        pendingRequests[originalObjectID].emplace_back(client, requestHeader);
+    }
+}
+
+void ObjectStorageServer::sendGetResponse(
+    std::shared_ptr<Client> client,
+    const ObjectRequestHeader& requestHeader,
+    std::shared_ptr<const ObjectPayload> objectPtr)
+{
+    uint64_t payloadLength = std::min(static_cast<uint64_t>(objectPtr->size()), requestHeader.payloadLength);
+
+    ObjectResponseHeader responseHeader {
+        .objectID      = requestHeader.objectID,
+        .payloadLength = payloadLength,
+        .responseID    = requestHeader.requestID,
+        .responseType  = ObjectResponseType::GET_O_K,
+    };
+
+    writeMessage(client, responseHeader, {objectPtr->data(), payloadLength});
+}
+
+void ObjectStorageServer::sendDuplicateResponse(
+    std::shared_ptr<Client> client, const ObjectRequestHeader& requestHeader)
+{
+    ObjectResponseHeader responseHeader {
+        .objectID      = requestHeader.objectID,
+        .payloadLength = 0,
+        .responseID    = requestHeader.requestID,
+        .responseType  = ObjectResponseType::DUPLICATE_O_K,
+    };
+
+    writeMessage(client, responseHeader, {});
+}
+
+void ObjectStorageServer::optionallySendPendingRequests(
+    const ObjectID& objectID, std::shared_ptr<const ObjectPayload> objectPtr)
+{
+    auto it = pendingRequests.find(objectID);
+    if (it == pendingRequests.end()) {
+        return;
+    }
+
+    // Immediately remove the object's pending requests, or else another coroutine might process them too.
+    auto requests = std::move(it->second);
+    pendingRequests.erase(it);
+
+    std::vector<ObjectStorageServer::SendMessageFuture> res;
+    for (auto& request: requests) {
+        if (request.requestHeader.requestType == ObjectRequestType::GET_OBJECT) {
+            sendGetResponse(request.client, request.requestHeader, objectPtr);
+        } else {
+            assert(request.requestHeader.requestType == ObjectRequestType::DUPLICATE_OBJECT_I_D);
+            objectManager.duplicateObject(objectID, request.requestHeader.objectID);
+            sendDuplicateResponse(request.client, request.requestHeader);
+
+            // Some other pending requests might be themselves dependent on this duplicated object.
+            optionallySendPendingRequests(request.requestHeader.objectID, objectPtr);
+        }
+    }
+    return;
+}
+
+};  // namespace object_storage
+};  // namespace scaler

scaler/object_storage/object_storage_server.h

@@ -0,0 +1,126 @@
+#pragma once
+
+#include <expected>
+#include <iostream>
+#include <memory>
+#include <optional>
+#include <span>
+
+#include "scaler/io/ymq/configuration.h"
+#include "scaler/io/ymq/io_context.h"
+#include "scaler/io/ymq/io_socket.h"
+#include "scaler/io/ymq/logging.h"
+#include "scaler/io/ymq/simple_interface.h"
+#include "scaler/object_storage/constants.h"
+#include "scaler/object_storage/defs.h"
+#include "scaler/object_storage/io_helper.h"
+#include "scaler/object_storage/message.h"
+#include "scaler/object_storage/object_manager.h"
+
+namespace scaler {
+namespace object_storage {
+
+class ObjectStorageServer {
+public:
+    using Identity          = ymq::Configuration::IOSocketIdentity;
+    using SendMessageFuture = std::future<std::optional<ymq::Error>>;
+
+    ObjectStorageServer();
+
+    ~ObjectStorageServer();
+
+    void run(
+        std::string name,
+        std::string port,
+        Identity identity                  = "ObjectStorageServer",
+        std::string log_level              = "INFO",
+        std::string log_format             = "%(levelname)s: %(message)s",
+        std::vector<std::string> log_paths = {"/dev/stdout"});
+
+    void waitUntilReady();
+
+    void shutdown();
+
+private:
+    struct Client {
+        std::shared_ptr<ymq::IOSocket> _ioSocket;
+        Identity _identity;
+    };
+
+    struct PendingRequest {
+        std::shared_ptr<Client> client;
+        ObjectRequestHeader requestHeader;
+    };
+
+    using ObjectRequestType  = scaler::protocol::ObjectRequestHeader::ObjectRequestType;
+    using ObjectResponseType = scaler::protocol::ObjectResponseHeader::ObjectResponseType;
+
+    ymq::IOContext _ioContext;
+    std::shared_ptr<ymq::IOSocket> _ioSocket;
+
+    int onServerReadyReader;
+    int onServerReadyWriter;
+
+    ObjectManager objectManager;
+
+    // Some GET and DUPLICATE requests might be delayed if the referenced object isn't available yet.
+    std::map<ObjectID, std::vector<PendingRequest>> pendingRequests;
+
+    scaler::ymq::Logger _logger;
+
+    std::atomic<bool> _stopped;
+
+    std::vector<SendMessageFuture> _pendingSendMessageFuts;
+
+    void initServerReadyFds();
+
+    void setServerReadyFd();
+
+    void closeServerReadyFds();
+
+    void processRequests();
+
+    void processSetRequest(std::shared_ptr<Client> client, std::pair<ObjectRequestHeader, Bytes> request);
+
+    void processGetRequest(std::shared_ptr<Client> client, const ObjectRequestHeader& requestHeader);
+
+    void processDeleteRequest(std::shared_ptr<Client> client, ObjectRequestHeader& requestHeader);
+
+    void processDuplicateRequest(std::shared_ptr<Client> client, std::pair<ObjectRequestHeader, Bytes> request);
+
+    template <ObjectStorageMessage T>
+    void writeMessage(std::shared_ptr<Client> client, T& message, std::span<const unsigned char> payload)
+    {
+        // Send OSS header
+        auto messageBuffer = message.toBuffer();
+        ymq::Message ymqHeader {};
+        ymqHeader.address     = Bytes(client->_identity);
+        ymqHeader.payload     = Bytes((char*)messageBuffer.asBytes().begin(), messageBuffer.asBytes().size());
+        auto sendHeaderFuture = ymq::futureSendMessage(client->_ioSocket, std::move(ymqHeader));
+
+        if (!payload.data()) {
+            _pendingSendMessageFuts.emplace_back(std::move(sendHeaderFuture));
+            return;
+        }
+
+        ymq::Message ymqPayload {};
+        ymqPayload.address     = Bytes(client->_identity);
+        ymqPayload.payload     = Bytes((char*)payload.data(), payload.size());
+        auto sendPayloadFuture = ymq::futureSendMessage(client->_ioSocket, std::move(ymqPayload));
+
+        _pendingSendMessageFuts.emplace_back(std::move(sendHeaderFuture));
+        _pendingSendMessageFuts.emplace_back(std::move(sendPayloadFuture));
+    }
+
+    void sendGetResponse(
+        std::shared_ptr<Client> client,
+        const ObjectRequestHeader& requestHeader,
+        std::shared_ptr<const ObjectPayload> objectPtr);
+
+    void sendDuplicateResponse(std::shared_ptr<Client> client, const ObjectRequestHeader& requestHeader);
+
+    void optionallySendPendingRequests(const ObjectID& objectID, std::shared_ptr<const ObjectPayload> objectPtr);
+};
+
+};  // namespace object_storage
+};  // namespace scaler

scaler/object_storage/pymod_object_storage_server.cpp

@@ -0,0 +1,104 @@
+#define PY_SSIZE_T_CLEAN
+#include <Python.h>
+
+#include "scaler/object_storage/object_storage_server.h"
+
+extern "C" {
+struct PyObjectStorageServer {
+    PyObject_HEAD scaler::object_storage::ObjectStorageServer server;
+};
+
+static PyObject* PyObjectStorageServerNew(
+    PyTypeObject* type, [[maybe_unused]] PyObject* args, [[maybe_unused]] PyObject* kwargs)
+{
+    PyObjectStorageServer* self;
+    self = (PyObjectStorageServer*)type->tp_alloc(type, 0);
+    return (PyObject*)self;
+}
+
+static int PyObjectStorageServerInit(PyObject* self, [[maybe_unused]] PyObject* args, [[maybe_unused]] PyObject* kwargs)
+{
+    new (&((PyObjectStorageServer*)self)->server) scaler::object_storage::ObjectStorageServer();
+    return 0;
+}
+
+static void PyObjectStorageServerDealloc(PyObject* self)
+{
+    ((PyObjectStorageServer*)self)->server.~ObjectStorageServer();
+    Py_TYPE(self)->tp_free((PyObject*)self);
+}
+
+static PyObject* PyObjectStorageServerRun(PyObject* self, PyObject* args)
+{
+    const char* addr;
+    int port;
+    const char* identity;
+    const char* log_level;
+    const char* log_format;
+    PyObject* logging_paths_tuple = nullptr;
+
+    if (!PyArg_ParseTuple(
+            args, "sisssO!", &addr, &port, &identity, &log_level, &log_format, &PyTuple_Type, &logging_paths_tuple))
+        return nullptr;
+
+    std::vector<std::string> logging_paths;
+    Py_ssize_t num_paths = PyTuple_Size(logging_paths_tuple);
+    for (Py_ssize_t i = 0; i < num_paths; ++i) {
+        PyObject* path_obj = PyTuple_GetItem(logging_paths_tuple, i);
+        if (!PyUnicode_Check(path_obj)) {
+            PyErr_SetString(PyExc_TypeError, "logging_paths must be a tuple of strings");
+            return nullptr;
+        }
+        logging_paths.push_back(PyUnicode_AsUTF8(path_obj));
+    }
+
+    ((PyObjectStorageServer*)self)
+        ->server.run(addr, std::to_string(port), identity, log_level, log_format, std::move(logging_paths));
+
+    Py_RETURN_NONE;
+}
+
+static PyObject* PyObjectStorageServerWaitUntilReady(PyObject* self, [[maybe_unused]] PyObject* args)
+{
+    ((PyObjectStorageServer*)self)->server.waitUntilReady();
+    Py_RETURN_NONE;
+}
+
+static PyMethodDef PyObjectStorageServerMethods[] = {
+    {"run", PyObjectStorageServerRun, METH_VARARGS, "Run object storage server on address:port with logging config"},
+    {"wait_until_ready", PyObjectStorageServerWaitUntilReady, METH_NOARGS, "Wait until the server is ready"},
+    {nullptr, nullptr, 0, nullptr},
+};
+
+static PyTypeObject PyObjectStorageServerType = {
+    .tp_name      = "object_storage_server.ObjectStorageServer",
+    .tp_basicsize = sizeof(PyObjectStorageServer),
+    .tp_dealloc   = (destructor)PyObjectStorageServerDealloc,
+    .tp_flags     = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE,
+    .tp_doc       = "ObjectStorageServer",
+    .tp_methods   = PyObjectStorageServerMethods,
+    .tp_init      = (initproc)PyObjectStorageServerInit,
+    .tp_new       = PyObjectStorageServerNew,
+};
+
+static PyModuleDef PyObjectStorageServerModule = {
+    .m_base = PyModuleDef_HEAD_INIT,
+    .m_name = "object_storage_server",
+    .m_size = -1,
+};
+
+PyMODINIT_FUNC PyInit_object_storage_server(void)
+{
+    PyObject* m;
+    if (PyType_Ready(&PyObjectStorageServerType) < 0)
+        return nullptr;
+
+    m = PyModule_Create(&PyObjectStorageServerModule);
+    if (m == nullptr)
+        return nullptr;
+
+    Py_INCREF(&PyObjectStorageServerType);
+    PyModule_AddObject(m, "ObjectStorageServer", (PyObject*)&PyObjectStorageServerType);
+    return m;
+}
+}

scaler/protocol/capnp/_python.py

@@ -0,0 +1,6 @@
+import capnp  # noqa
+
+import scaler.protocol.capnp.common_capnp as _common  # noqa
+import scaler.protocol.capnp.message_capnp as _message  # noqa
+import scaler.protocol.capnp.object_storage_capnp as _object_storage  # noqa
+import scaler.protocol.capnp.status_capnp as _status  # noqa

scaler/protocol/capnp/common.capnp

@@ -0,0 +1,63 @@
+@0xf57f79ac88fab620;
+
+enum TaskResultType {
+    success @0;           # if submit and task is done and get result
+    failed @1;            # if submit and task is failed on worker
+    failedWorkerDied @2;  # if submit and worker died (only happened when scheduler keep_task=False)
+}
+
+enum TaskCancelConfirmType {
+    canceled @0;               # if cancel success
+    cancelFailed @1;           # if cancel failed, this might happened if the task is in process
+    cancelNotFound @2;         # if cancel cannot find such task
+}
+
+enum TaskTransition {
+    hasCapacity @0;
+    taskResultSuccess @1;
+    taskResultFailed @2;
+    taskResultWorkerDied @3;
+    taskCancel @4;
+    taskCancelConfirmCanceled @5;
+    taskCancelConfirmFailed @6;
+    taskCancelConfirmNotFound @7;
+    balanceTaskCancel @8;
+    workerDisconnect @9;
+    schedulerHasTask @10;
+    schedulerHasNoTask @11;
+}
+
+enum TaskState {
+    inactive @0;
+    running @1;
+    canceling @2;
+    balanceCanceling @3;
+    success @4;
+    failed @5;
+    failedWorkerDied @6;
+    canceled @7;
+    canceledNotFound @8;
+    balanceCanceled @9;
+    workerDisconnecting @10;
+}
+
+struct TaskCapability {
+    name @0 :Text;      # the name of the capability provided by the worker/required by the task (e.g. "gpu" or "linux")
+    value @1 :Int64;    # the quantity of the capability provided/required. Use -1 for quantity-less capabilities
+}
+
+struct ObjectMetadata {
+    objectIds @0 :List(Data);
+    objectTypes @1 :List(ObjectContentType);
+    objectNames @2 :List(Data);
+
+    enum ObjectContentType {
+        serializer @0;
+        object @1;
+    }
+}
+
+struct ObjectStorageAddress {
+    host @0 :Text;
+    port @1 :UInt16;
+}