quasardb 3.14.1.dev4__cp39-cp39-macosx_10_14_x86_64.whl → 3.14.1.dev6__cp39-cp39-macosx_10_14_x86_64.whl

quasardb/CMakeLists.txt

@@ -325,6 +325,7 @@ set(QDB_FILES
   batch_inserter.hpp
   blob.hpp
   cluster.hpp
+  cluster.cpp
   concepts.hpp
   continuous.cpp
   continuous.hpp
@@ -333,9 +334,12 @@ set(QDB_FILES
   error.hpp
   entry.hpp
   handle.hpp
+  handle.cpp
   logger.hpp
   logger.cpp
   masked_array.hpp
+  metrics.hpp
+  metrics.cpp
   module.hpp
   module.cpp
   node.hpp

quasardb/cluster.cpp

@@ -0,0 +1,89 @@
+#include "cluster.hpp"
+#include "metrics.hpp"
+#include <chrono>
+#include <thread>
+
+namespace qdb
+{
+
+cluster::cluster(const std::string & uri,
+    const std::string & user_name,
+    const std::string & user_private_key,
+    const std::string & cluster_public_key,
+    const std::string & user_security_file,
+    const std::string & cluster_public_key_file,
+    std::chrono::milliseconds timeout,
+    bool do_version_check)
+    : _uri{uri}
+    , _handle{make_handle_ptr()}
+    , _json_loads{pybind11::module::import("json").attr("loads")}
+    , _logger("quasardb.cluster")
+{
+    if (do_version_check == true)
+    {
+        _logger.warn(
+            "do_version_check parameter has been deprecated and a no-op. It will be removed from a "
+            "future release");
+    }
+
+    options().apply_credentials(user_name, user_private_key, cluster_public_key, //
+        user_security_file, cluster_public_key_file);
+
+    options().set_timeout(timeout);
+
+    // HACKS(leon): we need to ensure there is always one callback active
+    //              for qdb. Callbacks can be lost when the last active session
+    //              gets closed. As such, the most pragmatic place to 'check'
+    //              for this callback is when establishing a new connection.
+    qdb::native::swap_callback();
+
+    _logger.info("Connecting to cluster %s", _uri);
+    _handle->connect(_uri);
+}
+
+void cluster::close()
+{
+    _logger.info("Closing connection to cluster");
+
+    try
+    {
+        if (is_open() == true) [[likely]]
+        {
+            _handle->close();
+        }
+    }
+    catch (qdb::invalid_handle_exception const & e)
+    {
+        // This can happen if, for example, we call close() after an error occured; in those
+        // circumstances, we fully expect the connection to already be invalid, and we should
+        // not care if this specific exception is raised.
+        _logger.warn("Connection already closed");
+    }
+
+    _handle.reset();
+
+    assert(is_open() == false);
+}
+
+void cluster::wait_for_compaction()
+{
+
+    // We define this function in the .cpp file so we can avoid including chrono and thread
+    // in the header file.
+
+    using namespace std::chrono_literals;
+
+    for (;;)
+    {
+        std::uint64_t progress = compact_progress();
+
+        if (progress == 0) [[unlikely]]
+        {
+            break;
+        }
+
+        std::this_thread::sleep_for(100ms);
+    }
+}
+
+}; // namespace qdb

quasardb/cluster.hpp

@@ -57,6 +57,7 @@
 #include <pybind11/operators.h>
 #include <pybind11/stl.h>
 #include <chrono>
+#include <iostream>
 
 namespace qdb
 {
@@ -71,39 +72,30 @@ public:
         const std::string & user_security_file      = {},
         const std::string & cluster_public_key_file = {},
         std::chrono::milliseconds timeout           = std::chrono::minutes{1},
-        bool do_version_check                       = false)
-        : _uri{uri}
-        , _handle{make_handle_ptr()}
-        , _json_loads{pybind11::module::import("json").attr("loads")}
-        , _logger("quasardb.cluster")
-    {
-        if (do_version_check == true)
-        {
-            _logger.warn(
-                "do_version_check parameter has been deprecated and a no-op. It will be removed from a "
-                "future release");
-        }
-
-        options().apply_credentials(user_name, user_private_key, cluster_public_key, //
-            user_security_file, cluster_public_key_file);
+        bool do_version_check                       = false);
 
-        options().set_timeout(timeout);
-
-        // HACKS(leon): we need to ensure there is always one callback active
-        //              for qdb. Callbacks can be lost when the last active session
-        //              gets closed. As such, the most pragmatic place to 'check'
-        //              for this callback is when establishing a new connection.
-        qdb::native::swap_callback();
+public:
+    void close();
 
-        _logger.info("Connecting to cluster %s", _uri);
-        _handle->connect(_uri);
+    bool is_open() const
+    {
+        return _handle.get() != nullptr && _handle->is_open();
     }
 
-public:
-    void close()
+    /**
+     * Throws exception if the connection is not open. Should be invoked before any operation
+     * is done on the handle, as the QuasarDB C API only checks for a canary presence in the
+     * handle's memory arena. If a compiler is optimizing enough, the handle can be closed but
+     * the canary still present in memory, so it's UB.
+     *
+     * As such, we should check on a higher level.
+     */
+    void check_open() const
     {
-        _logger.info("Closing connection to cluster");
-        _handle.reset();
+        if (is_open() == false) [[unlikely]]
+        {
+            throw qdb::invalid_handle_exception{};
+        }
     }
 
     void tidy_memory()
@@ -153,11 +145,6 @@ public:
         return *this;
     }
 
-    bool is_open() const
-    {
-        return _handle.get() != nullptr;
-    }
-
     void exit(pybind11::object type, pybind11::object value, pybind11::object traceback)
     {
         return close();
@@ -165,6 +152,8 @@ public:
 
     pybind11::object node_config(const std::string & uri)
     {
+        check_open();
+
         const char * content      = nullptr;
         qdb_size_t content_length = 0;
 
@@ -176,6 +165,8 @@ public:
 
     pybind11::object node_status(const std::string & uri)
     {
+        check_open();
+
         const char * content      = nullptr;
         qdb_size_t content_length = 0;
 
@@ -187,6 +178,8 @@ public:
 
     pybind11::object node_topology(const std::string & uri)
     {
+        check_open();
+
         const char * content      = nullptr;
         qdb_size_t content_length = 0;
 
@@ -199,64 +192,88 @@ public:
 public:
     qdb::tag tag(const std::string & alias)
     {
+        check_open();
+
         return qdb::tag{_handle, alias};
     }
 
     qdb::blob_entry blob(const std::string & alias)
     {
+        check_open();
+
         return qdb::blob_entry{_handle, alias};
     }
 
     qdb::string_entry string(const std::string & alias)
     {
+        check_open();
+
         return qdb::string_entry{_handle, alias};
     }
 
     qdb::integer_entry integer(const std::string & alias)
     {
+        check_open();
+
         return qdb::integer_entry{_handle, alias};
     }
 
     qdb::double_entry double_(const std::string & alias)
     {
+        check_open();
+
         return qdb::double_entry{_handle, alias};
     }
 
     qdb::timestamp_entry timestamp(const std::string & alias)
     {
+        check_open();
+
         return qdb::timestamp_entry{_handle, alias};
     }
 
     qdb::table table(const std::string & alias)
     {
+        check_open();
+
         return qdb::table{_handle, alias};
     }
 
     // the batch_inserter_ptr is non-copyable
     qdb::batch_inserter_ptr inserter(const std::vector<batch_column_info> & ci)
     {
+        check_open();
+
         return std::make_unique<qdb::batch_inserter>(_handle, ci);
     }
 
     // the batch_inserter_ptr is non-copyable
     qdb::writer_ptr writer()
     {
+        check_open();
+
         return std::make_unique<qdb::writer>(_handle);
     }
 
     // the batch_inserter_ptr is non-copyable
     qdb::writer_ptr pinned_writer()
     {
+        check_open();
+
         return writer();
     }
 
     qdb::options options()
     {
+        check_open();
+
         return qdb::options{_handle};
     }
 
     qdb::perf perf()
     {
+        check_open();
+
         return qdb::perf{_handle};
     }
 
@@ -268,6 +285,8 @@ public:
 public:
     std::vector<std::string> prefix_get(const std::string & prefix, qdb_int_t max_count)
     {
+        check_open();
+
         const char ** result = nullptr;
         size_t count         = 0;
 
@@ -283,6 +302,8 @@ public:
 
     qdb_uint_t prefix_count(const std::string & prefix)
     {
+        check_open();
+
         qdb_uint_t count = 0;
 
         const qdb_error_t err = qdb_prefix_count(*_handle, prefix.c_str(), &count);
@@ -294,22 +315,30 @@ public:
 public:
     qdb::find_query find(const std::string & query_string)
     {
+        check_open();
+
         return qdb::find_query{_handle, query_string};
     }
 
     py::object query(const std::string & query_string, const py::object & blobs)
     {
+        check_open();
+
         return py::cast(qdb::dict_query(_handle, query_string, blobs));
     }
 
     py::object query_numpy(const std::string & query_string)
     {
+        check_open();
+
         return py::cast(qdb::numpy_query(_handle, query_string));
     }
 
     std::shared_ptr<qdb::query_continuous> query_continuous_full(
         const std::string & query_string, std::chrono::milliseconds pace, const py::object & blobs)
     {
+        check_open();
+
         return std::make_shared<qdb::query_continuous>(
             _handle, qdb_query_continuous_full, pace, query_string, blobs);
     }
@@ -317,6 +346,8 @@ public:
     std::shared_ptr<qdb::query_continuous> query_continuous_new_values(
         const std::string & query_string, std::chrono::milliseconds pace, const py::object & blobs)
     {
+        check_open();
+
         return std::make_shared<qdb::query_continuous>(
             _handle, qdb_query_continuous_new_values_only, pace, query_string, blobs);
     }
@@ -324,6 +355,8 @@ public:
 public:
     std::vector<std::string> suffix_get(const std::string & suffix, qdb_int_t max_count)
     {
+        check_open();
+
         const char ** result = nullptr;
         size_t count         = 0;
 
@@ -339,6 +372,8 @@ public:
 
     qdb_uint_t suffix_count(const std::string & suffix)
     {
+        check_open();
+
         qdb_uint_t count = 0;
 
         const qdb_error_t err = qdb_suffix_count(*_handle, suffix.c_str(), &count);
@@ -350,31 +385,72 @@ public:
 public:
     void purge_all(std::chrono::milliseconds timeout_ms)
     {
+        check_open();
+
         qdb::qdb_throw_if_error(
             *_handle, qdb_purge_all(*_handle, static_cast<int>(timeout_ms.count())));
     }
 
     void purge_cache(std::chrono::milliseconds timeout_ms)
     {
+        check_open();
+
         qdb::qdb_throw_if_error(
             *_handle, qdb_purge_cache(*_handle, static_cast<int>(timeout_ms.count())));
     }
 
     void wait_for_stabilization(std::chrono::milliseconds timeout_ms)
     {
+        check_open();
+
         qdb::qdb_throw_if_error(
             *_handle, qdb_wait_for_stabilization(*_handle, static_cast<int>(timeout_ms.count())));
     }
 
     void trim_all(std::chrono::milliseconds pause_ms, std::chrono::milliseconds timeout_ms)
     {
+        check_open();
+
         qdb::qdb_throw_if_error(*_handle, qdb_trim_all(*_handle, static_cast<int>(pause_ms.count()),
                                               static_cast<int>(timeout_ms.count())));
     }
 
+    void compact_full()
+    {
+        check_open();
+
+        qdb_compact_params_t params{};
+        params.options = qdb_compact_full;
+
+        qdb::qdb_throw_if_error(*_handle, qdb_cluster_compact(*_handle, &params));
+    }
+
+    // Returns 0 when finished / no compaction running
+    std::uint64_t compact_progress()
+    {
+        check_open();
+
+        std::uint64_t progress;
+
+        qdb::qdb_throw_if_error(*_handle, qdb_cluster_get_compact_progress(*_handle, &progress));
+
+        return progress;
+    }
+
+    void compact_abort()
+    {
+        check_open();
+
+        qdb::qdb_throw_if_error(*_handle, qdb_cluster_abort_compact(*_handle));
+    }
+
+    void wait_for_compaction();
+
 public:
     std::vector<std::string> endpoints()
     {
+        check_open();
+
         qdb_remote_node_t * endpoints = nullptr;
         qdb_size_t count              = 0;
 
@@ -465,6 +541,10 @@ static inline void register_cluster(Module & m)
         .def("purge_all", &qdb::cluster::purge_all)                                           //
         .def("trim_all", &qdb::cluster::trim_all)                                             //
         .def("purge_cache", &qdb::cluster::purge_cache)                                       //
+        .def("compact_full", &qdb::cluster::compact_full)                                     //
+        .def("compact_progress", &qdb::cluster::compact_progress)                             //
+        .def("compact_abort", &qdb::cluster::compact_abort)                                   //
+        .def("wait_for_compaction", &qdb::cluster::wait_for_compaction)                       //
         .def("endpoints", &qdb::cluster::endpoints);                                          //
 }
 

quasardb/cmake_install.cmake

@@ -34,7 +34,7 @@ endif()
 
 # Set default install directory permissions.
 if(NOT DEFINED CMAKE_OBJDUMP)
-  set(CMAKE_OBJDUMP "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/objdump")
+  set(CMAKE_OBJDUMP "/usr/local/clang16/bin/llvm-objdump")
 endif()
 
 if(NOT CMAKE_INSTALL_LOCAL_ONLY)

quasardb/date/cmake_install.cmake

@@ -34,7 +34,7 @@ endif()
 
 # Set default install directory permissions.
 if(NOT DEFINED CMAKE_OBJDUMP)
-  set(CMAKE_OBJDUMP "/Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/bin/objdump")
+  set(CMAKE_OBJDUMP "/usr/local/clang16/bin/llvm-objdump")
 endif()
 
 if(CMAKE_INSTALL_COMPONENT STREQUAL "Unspecified" OR NOT CMAKE_INSTALL_COMPONENT)

quasardb/error.hpp

@@ -78,6 +78,17 @@ public:
     {}
 };
 
+class invalid_handle_exception : public exception
+{
+public:
+    invalid_handle_exception() noexcept
+        : exception(qdb_e_invalid_handle,
+            std::string("Invalid handle: the connection to the cluster has been closed. Please "
+                        "re-establish a new connection"
+                        "with the QuasarDB cluster."))
+    {}
+};
+
 class incompatible_type_exception : public exception
 {
 public:
@@ -218,7 +229,15 @@ inline void qdb_throw_if_error(
         {
             // Error context returned is not the same, which means this thread already made
             // another call to the QDB API, or the QDB API itself
-            throw qdb::exception{err, qdb_error(err)};
+
+            switch (err)
+            {
+            case qdb_e_not_connected:
+            case qdb_e_invalid_handle:
+                throw qdb::invalid_handle_exception{};
+            default:
+                throw qdb::exception{err, qdb_error(err)};
+            }
         }
         assert(err_ == err);
         assert(msg_ != nullptr);
@@ -233,6 +252,10 @@ inline void qdb_throw_if_error(
         case qdb_e_invalid_query:
             throw qdb::invalid_query_exception{msg_data_};
 
+        case qdb_e_not_connected:
+        case qdb_e_invalid_handle:
+            throw qdb::invalid_handle_exception{};
+
         case qdb_e_alias_already_exists:
             throw qdb::alias_already_exists_exception{msg_data_};
 
@@ -289,6 +312,7 @@ static inline void register_errors(Module & m)
     py::register_exception<qdb::internal_local_exception>(m, "InternalLocalError", base_class);
     py::register_exception<qdb::invalid_argument_exception>(m, "InvalidArgumentError", base_class);
     py::register_exception<qdb::invalid_query_exception>(m, "InvalidQueryError", base_class);
+    py::register_exception<qdb::invalid_handle_exception>(m, "InvalidHandleError", base_class);
 }
 
 } // namespace qdb

quasardb/handle.cpp

@@ -0,0 +1,29 @@
+#include "handle.hpp"
+#include "metrics.hpp"
+
+namespace qdb
+{
+
+void handle::connect(const std::string & uri)
+{
+    qdb_error_t err;
+    {
+        metrics::scoped_capture{"qdb_connect"};
+        err = qdb_connect(handle_, uri.c_str());
+    }
+    qdb::qdb_throw_if_error(handle_, err);
+}
+
+void handle::close()
+{
+    if (handle_ != nullptr)
+    {
+        metrics::scoped_capture{"qdb_close"};
+        qdb_close(handle_);
+        handle_ = nullptr;
+    }
+
+    assert(handle_ == nullptr);
+}
+
+}; // namespace qdb

quasardb/handle.hpp

@@ -42,33 +42,50 @@ class handle
 {
 public:
     handle() noexcept
+        : handle_{nullptr}
     {}
 
     explicit handle(qdb_handle_t h) noexcept
-        : _handle{h}
+        : handle_{h}
     {}
 
     ~handle()
     {
-        if (_handle)
-        {
-            qdb_close(_handle);
-            _handle = nullptr;
-        }
+        close();
     }
 
-    void connect(const std::string & uri)
+    void connect(const std::string & uri);
+
+    operator qdb_handle_t() const noexcept
     {
-        qdb::qdb_throw_if_error(_handle, qdb_connect(_handle, uri.c_str()));
+        return handle_;
     }
 
-    operator qdb_handle_t() const noexcept
+    void close();
+
+    constexpr inline bool is_open() const
     {
-        return _handle;
+        return handle_ != nullptr;
+    }
+
+    /**
+     * Throws exception if the connection is not open. Should be invoked before any operation
+     * is done on the handle, as the QuasarDB C API only checks for a canary presence in the
+     * handle's memory arena. If a compiler is optimizing enough, the handle can be closed but
+     * the canary still present in memory, so it's UB.
+     *
+     * As such, we should check on a higher level.
+     */
+    inline void check_open() const
+    {
+        if (is_open() == false) [[unlikely]]
+        {
+            throw qdb::invalid_handle_exception{};
+        }
     }
 
 private:
-    qdb_handle_t _handle{nullptr};
+    qdb_handle_t handle_{nullptr};
 };
 
 using handle_ptr = std::shared_ptr<handle>;

quasardb/logger.cpp

@@ -11,6 +11,8 @@ void qdb::native::swap_callback()
 {
     // Potential race condition avoidance!
     std::lock_guard<std::mutex> guard(_buffer_lock);
+
+#ifndef NDEBUG
     qdb_error_t error;
 
     error = qdb_log_remove_callback(local_callback_id);
@@ -26,6 +28,7 @@ void qdb::native::swap_callback()
         // fprintf(stderr, "unable to add new callback: %s (%#x)\n", qdb_error(error), error);
         // fflush(stderr);
     }
+#endif
 
     _logger = qdb::logger("quasardb.native");
 }