quasardb 3.14.2.dev0__cp37-cp37m-win_amd64.whl → 3.14.2.dev2__cp37-cp37m-win_amd64.whl

quasardb/detail/retry.hpp

@@ -0,0 +1,147 @@
+/*
+ *
+ * Official Python API
+ *
+ * Copyright (c) 2009-2021, quasardb SAS. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *    * Neither the name of quasardb nor the names of its contributors may
+ *      be used to endorse or promote products derived from this software
+ *      without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY QUASARDB AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#pragma once
+
+#include "../error.hpp"
+#include <chrono>
+
+namespace qdb::detail
+{
+
+struct retry_options
+{
+    qdb::logger logger_;
+
+    // how many retries are left. 0 means no retries
+    std::size_t retries_left;
+
+    // delay for the next retry
+    std::chrono::milliseconds delay;
+
+    // factor by which delay is increased every retry
+    std::size_t exponent;
+
+    // random jitter added/removed to delay_. Jitter of 0.1 means that 10% is automatically
+    // added or removed from delay_
+    double jitter;
+
+    /**
+     * Retry options constructor
+     */
+    retry_options(std::size_t retries   = 3,
+        std::chrono::milliseconds delay = std::chrono::milliseconds{3000},
+        std::size_t exponent            = 2,
+        double jitter                   = 0.1)
+        : logger_("quasardb.detail.retry")
+        , retries_left{retries}
+        , delay{delay}
+        , exponent{exponent}
+        , jitter{jitter}
+    {}
+
+    static inline retry_options from_kwargs(py::kwargs args)
+    {
+        if (args.contains("retries") == false)
+        {
+            return {};
+        }
+
+        auto retries = args["retries"];
+
+        // We're going to assume that retries is an actual RetryOptions class, because
+        // our Numpy and Pandas adapters always coerce it to that type. By making this
+        // assumption, we can slightly optimize this code path.
+        //
+        // By providing an explicit RetryOptions object is also the only way to fine-tune
+        // other parameters.
+
+        try
+        {
+            return retries.cast<retry_options>();
+        }
+        catch (py::cast_error const & /*e*/)
+        {
+            // For convenience, we also give the user the choice to just provide `retries`
+            // as an integer, in which case we'll assume this is the number of retries the
+            // user wants to perform.
+            //
+            // In all likelihood, this is the only parameter the user will want to tune.
+            return {retries.cast<std::size_t>()};
+        }
+    }
+
+    inline constexpr bool has_next() const noexcept
+    {
+        return retries_left > 0;
+    }
+
+    /**
+     * Returns new object, with `retries_left_` and `delay_` adjusted accordingly.
+     */
+    inline retry_options next() const
+    {
+        if (has_next() == false) [[unlikely]]
+        {
+            throw qdb::out_of_bounds_exception{
+                "RetryOptions.next() called but retries already exhausted."};
+        }
+
+        assert(has_next() == true);
+
+        return retry_options{retries_left - 1, delay * exponent, exponent, jitter};
+    }
+
+    /**
+     * Returns true if we have retries left and the error is retryable.
+     */
+    inline constexpr bool should_retry(qdb_error_t e) const noexcept
+    {
+        // We check for `is_retryable` first, because in all likelihood e is qdb_e_ok
+        return retry_options::is_retryable(e) && has_next();
+    }
+
+    static constexpr bool is_retryable(qdb_error_t e) noexcept
+    {
+        switch (e)
+        {
+        case qdb_e_async_pipe_full:
+        case qdb_e_try_again:
+            return true;
+            break;
+        default:
+            return false;
+        };
+    }
+};
+
+void register_retry_options(py::module_ & m);
+
+}; // namespace qdb::detail

quasardb/detail/sleep.hpp

@@ -0,0 +1,53 @@
+/*
+ *
+ * Official Python API
+ *
+ * Copyright (c) 2009-2021, quasardb SAS. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *    * Neither the name of quasardb nor the names of its contributors may
+ *      be used to endorse or promote products derived from this software
+ *      without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY QUASARDB AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#pragma once
+
+#include "../concepts.hpp"
+#include <chrono>
+
+namespace qdb::detail
+{
+
+template <typename Duration = std::chrono::milliseconds>
+struct default_sleep_strategy
+{
+    using duration_t = Duration;
+
+    static void sleep(duration_t const & delay)
+    {
+        std::this_thread::sleep_for(delay);
+    };
+};
+
+static_assert(qdb::concepts::sleep_strategy<default_sleep_strategy<>>);
+static_assert(qdb::concepts::sleep_strategy<default_sleep_strategy<std::chrono::seconds>>);
+
+}; // namespace qdb::detail

quasardb/{writer.cpp → detail/writer.cpp}

@@ -1,10 +1,10 @@
 #include "writer.hpp"
+#include "../convert/array.hpp"
 #include "concepts.hpp"
 #include "dispatch.hpp"
-#include "metrics.hpp"
 #include "numpy.hpp"
+#include "retry.hpp"
 #include "traits.hpp"
-#include "convert/array.hpp"
 
 namespace qdb::detail
 {
@@ -123,10 +123,10 @@ struct fill_column_dispatch
     {
         qdb_exp_batch_push_column_t ret{};
 
-        ret.name      = traits::null_value<qdb_string_t>();
+        ret.name      = nullptr;
         ret.data_type = ctype;
 
-        // We swap the pointer inside the `qdb_exp_batch_push_column__t` with the pointer
+        // We swap the pointer inside the `qdb_exp_batch_push_column_t` with the pointer
         // to the data as it is inside the `any_column`.
         //
         // This works, because all these `any_column`'s lifecycle is scoped to the
@@ -191,7 +191,7 @@ std::vector<qdb_exp_batch_push_column_t> const & staged_table::prepare_columns()
         // XXX(leon): reuses lifecycle of _column_infos[index], which *should* be fine,
         //            but ensuring we take a reference is super important here!
         std::string const & column_name = _column_infos[index].name;
-        column.name                     = qdb_string_t{column_name.data(), column_name.size()};
+        column.name                     = column_name.c_str();
 
 #ifndef NDEBUG
         // Symbol column is actually string data
@@ -222,7 +222,7 @@ void staged_table::prepare_batch(qdb_exp_batch_push_mode_t mode,
     qdb_ts_range_t * ranges,
     qdb_exp_batch_push_table_t & batch)
 {
-    batch.name = qdb_string_t{_table_name.data(), _table_name.size()};
+    batch.name = _table_name.c_str();
 
     prepare_table_data(batch.data);
     if (mode == qdb_exp_batch_push_truncate)
@@ -234,117 +234,94 @@ void staged_table::prepare_batch(qdb_exp_batch_push_mode_t mode,
     // Zero-initialize these
     batch.where_duplicate       = nullptr;
     batch.where_duplicate_count = 0;
-    batch.options               = qdb_exp_batch_option_standard;
+    batch.deduplication_mode    = qdb_exp_batch_deduplication_mode_disabled;
+    batch.creation              = qdb_exp_batch_dont_create;
 
     enum detail::deduplication_mode_t mode_ = deduplicate_options.mode_;
 
-    std::visit([&mode_, &batch](auto const & columns) { _set_push_options(mode_, columns, batch); },
+    std::visit(
+        [&mode_, &batch](auto const & columns) { _set_deduplication_mode(mode_, columns, batch); },
         deduplicate_options.columns_);
 }
 
-}; // namespace qdb::detail
-
-namespace qdb
-{
-
-void writer::data::append(
-    qdb::table const & table, py::handle const & index, py::list const & column_data)
+/* static */ py::kwargs batch_push_mode::ensure(py::kwargs kwargs)
 {
-    py::array index_ = numpy::array::ensure<traits::datetime64_ns_dtype>(index);
-
-    /**
-     * Additional check that all the data is actually of the same length, and data has been
-     * provided for each and every column.
-     */
-    if (column_data.size() != table.list_columns().size())
+    if (kwargs.contains(kw_push_mode) == false)
+        [[unlikely]] // Our pandas/numpy adapters always set this, so unlikely
     {
-        throw qdb::invalid_argument_exception{"data must be provided for every table column"};
-    }
 
-    for (py::handle const & data : column_data)
-    {
-        qdb::masked_array data_ = data.cast<qdb::masked_array>();
-        if (data_.size() != static_cast<std::size_t>(index_.size()))
-        {
-            throw qdb::invalid_argument_exception{
-                "every data array should be exactly the same length as the index array"};
-        }
+        kwargs[batch_push_mode::kw_push_mode] = batch_push_mode::default_push_mode;
     }
 
-    xs_.push_back(value_type{table, index_, column_data});
+    return kwargs;
 }
 
-void writer::push(writer::data const & data, py::kwargs args)
+/* static */ py::kwargs batch_push_mode::set(py::kwargs kwargs, qdb_exp_batch_push_mode_t push_mode)
 {
-    qdb::object_tracker::scoped_capture capture{_object_tracker};
-    staged_tables_t staged_tables = _stage_tables(data);
+    assert(kwargs.contains(kw_push_mode) == false);
+
+    kwargs[batch_push_mode::kw_push_mode] = push_mode;
 
-    _push_impl(staged_tables, qdb_exp_batch_push_transactional, _deduplicate_from_args(args));
+    return kwargs;
 }
 
-void writer::push_async(writer::data const & data, py::kwargs args)
+/* static */ qdb_exp_batch_push_mode_t batch_push_mode::from_kwargs(py::kwargs const & kwargs)
 {
-    qdb::object_tracker::scoped_capture capture{_object_tracker};
-    staged_tables_t staged_tables = _stage_tables(data);
+    assert(kwargs.contains(batch_push_mode::kw_push_mode));
 
-    _push_impl(staged_tables, qdb_exp_batch_push_async, _deduplicate_from_args(args));
+    return py::cast<qdb_exp_batch_push_mode_t>(kwargs[batch_push_mode::kw_push_mode]);
 }
 
-void writer::push_fast(writer::data const & data, py::kwargs args)
+/* static */ py::kwargs detail::batch_push_flags::ensure(py::kwargs kwargs)
 {
-    qdb::object_tracker::scoped_capture capture{_object_tracker};
-    staged_tables_t staged_tables = _stage_tables(data);
+    if (kwargs.contains(batch_push_flags::kw_write_through) == false)
+    {
+        kwargs[batch_push_flags::kw_write_through] = batch_push_flags::default_write_through;
+    }
 
-    _push_impl(staged_tables, qdb_exp_batch_push_fast, _deduplicate_from_args(args));
+    return kwargs;
 }
 
-void writer::push_truncate(writer::data const & data, py::kwargs args)
+/* static */ qdb_uint_t detail::batch_push_flags::from_kwargs(py::kwargs const & kwargs)
 {
-    qdb::object_tracker::scoped_capture capture{_object_tracker};
-    staged_tables_t staged_tables = _stage_tables(data);
-
-    auto deduplicate = _deduplicate_from_args(args);
+    assert(kwargs.contains(batch_push_flags::kw_write_through));
 
-    // Sanity check, this should be checked for in the python-side of things as well,
-    // but people can invoke this manually if they want.
-    if (!std::holds_alternative<bool>(deduplicate.columns_)
-        || std::get<bool>(deduplicate.columns_) != false) [[unlikely]]
-    {
-        throw qdb::invalid_argument_exception{"Cannot set `deduplicate` for push_truncate."};
-    };
+    // By default no flags are set
+    qdb_uint_t ret = qdb_exp_batch_push_flag_none;
 
-    // As we are actively removing data, let's add an additional check to ensure the user
-    // doesn't accidentally truncate his whole database without inserting anything.
-    if (data.empty()) [[unlikely]]
+    try
     {
-        throw qdb::invalid_argument_exception{"Writer is empty: you did not provide any rows to push."};
-    }
-
-    qdb_ts_range_t tr;
+        bool write_through = py::cast<bool>(kwargs["write_through"]);
 
-    if (args.contains("range"))
-    {
-        tr = convert::value<py::tuple, qdb_ts_range_t>(py::cast<py::tuple>(args["range"]));
-    }
-    else
-    {
-        // TODO(leon): support multiple tables for push truncate
-        if (staged_tables.size() != 1) [[unlikely]]
+        // Likely as it's the default, if not, this static assert should fail:
+        static_assert(batch_push_flags::default_write_through == true);
+        if (write_through == true) [[likely]]
         {
-            throw qdb::invalid_argument_exception{
-                "Writer push truncate only supports a single "
-                "table unless an explicit range is provided: you provided more than one table without"
-                " an explicit range."};
+            ret |= static_cast<qdb_uint_t>(qdb_exp_batch_push_flag_write_through);
         }
+    }
+    catch (py::cast_error const & /*e*/)
+    {
+        std::string error_msg = "Invalid argument provided for `write_through`: expected bool, got: ";
+        error_msg += py::str(py::type::of(kwargs["write_through"])).cast<std::string>();
 
-        detail::staged_table const & staged_table = staged_tables.cbegin()->second;
-        tr                                        = staged_table.time_range();
+        throw qdb::invalid_argument_exception{error_msg};
     }
 
-    _push_impl(staged_tables, qdb_exp_batch_push_truncate, deduplicate, &tr);
+    return ret;
 }
 
-detail::deduplicate_options writer::_deduplicate_from_args(py::kwargs args)
+/* static */ qdb_exp_batch_options_t detail::batch_options::from_kwargs(py::kwargs const & kwargs)
+{
+    auto kwargs_ = detail::batch_push_mode::ensure(kwargs);
+
+    return {
+        .mode       = detail::batch_push_mode::from_kwargs(kwargs_), //
+        .push_flags = detail::batch_push_flags::from_kwargs(kwargs_) //
+    };
+}
+
+/* static */ detail::deduplicate_options detail::deduplicate_options::from_kwargs(py::kwargs args)
 {
     if (!args.contains("deduplicate") || !args.contains("deduplication_mode"))
     {
@@ -390,11 +367,16 @@ detail::deduplicate_options writer::_deduplicate_from_args(py::kwargs args)
     throw qdb::invalid_argument_exception{error_msg};
 };
 
-/* static */ writer::staged_tables_t writer::_stage_tables(writer::data const & data)
+/* static */ staged_tables staged_tables::index(detail::writer_data const & data)
 {
-    staged_tables_t staged_tables;
+    // XXX(leon): this function could potentially be moved to e.g. a free
+    // function as it doesn't really depend upon anything in writer, but
+    // then we'll have to globally declare staged_tables_t and it gets a
+    // bit messy.
+
+    detail::staged_tables ret;
 
-    for (writer::data::value_type const & table_data : data.xs())
+    for (detail::writer_data::value_type const & table_data : data.xs())
     {
         qdb::table table     = table_data.table;
         py::array index      = table_data.index;
@@ -408,7 +390,7 @@ detail::deduplicate_options writer::_deduplicate_from_args(py::kwargs args)
                 "data must be provided for every column of the table."};
         }
 
-        detail::staged_table & staged_table = writer::_get_staged_table(table, staged_tables);
+        detail::staged_table & staged_table = ret.get_or_create(table);
 
         staged_table.set_index(index);
 
@@ -453,82 +435,6 @@ detail::deduplicate_options writer::_deduplicate_from_args(py::kwargs args)
         }
     }
 
-    return staged_tables;
+    return ret;
 }
-
-void writer::_push_impl(writer::staged_tables_t & staged_tables,
-    qdb_exp_batch_push_mode_t mode,
-    detail::deduplicate_options deduplicate_options,
-    qdb_ts_range_t * ranges)
-{
-    _handle->check_open();
-
-    if (staged_tables.empty())
-    {
-        throw qdb::invalid_argument_exception{"No data written to batch writer."};
-    }
-
-    std::vector<qdb_exp_batch_push_table_t> batch;
-    batch.assign(staged_tables.size(), qdb_exp_batch_push_table_t());
-
-    int cur = 0;
-    _logger.debug("writer::_push_impl");
-
-    for (auto pos = staged_tables.begin(); pos != staged_tables.end(); ++pos)
-    {
-        std::string const & table_name      = pos->first;
-        detail::staged_table & staged_table = pos->second;
-        auto & batch_table                  = batch.at(cur++);
-
-        staged_table.prepare_batch(mode, deduplicate_options, ranges, batch_table);
-
-        if (batch_table.data.column_count == 0) [[unlikely]]
-        {
-            throw qdb::invalid_argument_exception{
-                "Writer is empty: you did not provide any columns to push."};
-        }
-
-        _logger.debug("Pushing %d rows with %d columns in %s", batch_table.data.row_count,
-            batch_table.data.column_count, table_name);
-    }
-
-    // Make sure to measure the time it takes to do the actual push
-    qdb::metrics::scoped_capture capture{"qdb_batch_push"};
-
-    qdb::qdb_throw_if_error(
-        *_handle, qdb_exp_batch_push(*_handle, mode, batch.data(), nullptr, batch.size()));
-}
-
-void register_writer(py::module_ & m)
-{
-    namespace py = pybind11;
-
-    // Writer data
-    auto writer_data_c = py::class_<qdb::writer::data>{m, "WriterData"};
-    writer_data_c.def(py::init())
-        .def("append", &qdb::writer::data::append, py::arg("table"), py::arg("index"),
-            py::arg("column_data"), "Append new data")
-        .def("empty", &qdb::writer::data::empty, "Returns true if underlying data is empty");
-
-    // And the actual pinned writer
-    auto writer_c = py::class_<qdb::writer>{m, "Writer"};
-
-    // basic interface
-    writer_c.def(py::init<qdb::handle_ptr>()); //
-
-    writer_c.def_readwrite("_legacy_state", &qdb::writer::legacy_state_);
-
-    // push functions
-    writer_c
-        .def("push", &qdb::writer::push, "Regular batch push") //
-        .def("push_async", &qdb::writer::push_async,
-            "Asynchronous batch push that buffers data inside the QuasarDB daemon") //
-        .def("push_fast", &qdb::writer::push_fast,
-            "Fast, in-place batch push that is efficient when doing lots of small, incremental "
-            "pushes.") //
-        .def("push_truncate", &qdb::writer::push_truncate,
-            "Before inserting data, truncates any existing data. This is useful when you want your "
-            "insertions to be idempotent, e.g. in "
-            "case of a retry.");
-}
-}; // namespace qdb
+}; // namespace qdb::detail