quasardb 3.14.2.dev1__cp311-cp311-macosx_11_0_arm64.whl → 3.14.2.dev2__cp311-cp311-macosx_11_0_arm64.whl

quasardb/reader.hpp

@@ -0,0 +1,256 @@
+/*
+ *
+ * Official Python API
+ *
+ * Copyright (c) 2009-2021, quasardb SAS. All rights reserved.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ *    * Redistributions of source code must retain the above copyright
+ *      notice, this list of conditions and the following disclaimer.
+ *    * Redistributions in binary form must reproduce the above copyright
+ *      notice, this list of conditions and the following disclaimer in the
+ *      documentation and/or other materials provided with the distribution.
+ *    * Neither the name of quasardb nor the names of its contributors may
+ *      be used to endorse or promote products derived from this software
+ *      without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY QUASARDB AND CONTRIBUTORS ``AS IS'' AND ANY
+ * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+ * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE REGENTS AND CONTRIBUTORS BE LIABLE FOR ANY
+ * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+#pragma once
+
+#include "handle.hpp"
+#include "logger.hpp"
+#include "object_tracker.hpp"
+#include <qdb/ts.h>
+#include <unordered_map>
+#include <vector>
+
+namespace py = pybind11;
+
+namespace qdb
+{
+
+namespace detail
+{
+
+using int64_column     = std::vector<qdb_int_t>;
+using double_column    = std::vector<double>;
+using timestamp_column = std::vector<qdb_timespec_t>;
+using blob_column      = std::vector<qdb_blob_t>;
+using string_column    = std::vector<qdb_string_t>;
+
+class reader_data
+{
+public:
+    /**
+     * Utility function which converts table data into a vanilla dict. Currently this works well, as
+     * there isn't any additional data/state we need to keep track of --
+     */
+    static py::dict convert(qdb_bulk_reader_table_data_t const & data);
+};
+
+class reader_iterator
+{
+public:
+    // Default constructor, which represents the "end" of the range
+    reader_iterator() noexcept
+        : handle_{nullptr}
+        , reader_{nullptr}
+        , batch_size_{0}
+        , table_count_{0}
+        , ptr_{nullptr}
+        , n_{0}
+    {}
+
+    // Actual initialization
+    reader_iterator(
+        handle_ptr handle, qdb_reader_handle_t reader, std::size_t batch_size, std::size_t table_count)
+        : handle_{handle}
+        , reader_{reader}
+        , batch_size_{batch_size}
+        , table_count_{table_count}
+
+        , ptr_{nullptr}
+        , n_{0}
+    {
+        // Always immediately try to fetch the first batch.
+        this->operator++();
+    }
+
+    bool operator!=(reader_iterator const & rhs) const noexcept
+    {
+        return !(*this == rhs);
+    }
+
+    bool operator==(reader_iterator const & rhs) const noexcept
+    {
+        // This is just a sanity check: if our handle_ is null, it means basically
+        // the entire object has to be null, and this will basically represent the
+        // ".end()" iterator.
+
+        if (handle_ == nullptr)
+        {
+            assert(reader_ == nullptr);
+            assert(ptr_ == nullptr);
+        }
+        else
+        {
+            assert(reader_ != nullptr);
+            assert(ptr_ != nullptr);
+        }
+
+        // Optimization: we *only* compare the pointers, we don't actually compare
+        // the data itself. This saves a bazillion comparisons, and for the purpose
+        // of iterators, we really only care whether the current iterator is at the
+        // end.
+        return (handle_ == rhs.handle_              //
+                && reader_ == rhs.reader_           //
+                && batch_size_ == rhs.batch_size_   //
+                && table_count_ == rhs.table_count_ //
+                && ptr_ == rhs.ptr_ && n_ == rhs.n_);
+    }
+
+    reader_iterator & operator++();
+
+    py::dict operator*()
+    {
+        assert(ptr_ != nullptr);
+        assert(n_ < table_count_);
+
+        return reader_data::convert(ptr_[n_]);
+    }
+
+private:
+    qdb::handle_ptr handle_;
+    qdb_reader_handle_t reader_;
+
+    /**
+     * The amount of rows to fetch in one operation. This can span multiple tables.
+     */
+    std::size_t batch_size_;
+
+    /**
+     * `table_count_` enables us to manage how much far we can iterate `ptr_`.
+     */
+    std::size_t table_count_;
+    qdb_bulk_reader_table_data_t * ptr_;
+    std::size_t n_;
+};
+
+}; // namespace detail
+
+class reader
+{
+public:
+    using iterator = detail::reader_iterator;
+
+public:
+    /**
+     * Tables must always be a list of actual table objects. This ensures the lifetime
+     * of any metadata inside the tables (such as its name) will always exceed that
+     * of the reader, which simplifies things a lot.
+     */
+    reader(                                            //
+        qdb::handle_ptr handle,                        //
+        std::vector<std::string> const & table_names,  //
+        std::vector<std::string> const & column_names, //
+        std::size_t batch_size,                        //
+        std::vector<py::tuple> const & ranges)         //
+        : logger_("quasardb.reader")
+        , handle_{handle}
+        , reader_{nullptr}
+        , table_names_{table_names}
+        , column_names_{column_names}
+        , batch_size_{batch_size}
+        , ranges_{ranges}
+    {}
+
+    // prevent copy because of the table object, use a unique_ptr of the batch in cluster
+    // to return the object.
+    //
+    // we prevent these copies because that is almost never what you want, and it gives us
+    // more freedom in storing a lot of data inside this object.
+    reader(const reader &) = delete;
+    reader(reader &&)      = delete;
+
+    ~reader()
+    {
+        close();
+    }
+
+    /**
+     * Convenience function for accessing the configured batch size. Returns 0 when everything should
+     * be read in a single batch.
+     */
+    constexpr inline std::size_t get_batch_size() const noexcept
+    {
+        return batch_size_;
+    }
+
+    /**
+     * Opens the actual reader; this will initiatate a call to quasardb and initialize the local
+     * reader handle. If table strings are provided instead of qdb::table objects, will automatically
+     * look those up.
+     *
+     * May throw exception upon error.
+     *
+     * :NOTE(leon): We just return a reference to ourselves, but maybe we want the outer object to wrap
+     *              a subclass and return that as well. Not 100% sure if that's the best way to go. This
+     *              works right now and is the same approach that we take with e.g. qdb::cluster
+     */
+    reader const & enter();
+
+    void exit(pybind11::object type, pybind11::object value, pybind11::object traceback)
+    {
+        return close();
+    }
+
+    /**
+     * Clean up and close. Does not require all data to be actually read.
+     */
+    void close();
+
+    iterator begin() const
+    {
+        if (reader_ == nullptr) [[unlikely]]
+        {
+            throw qdb::uninitialized_exception{
+                "Reader not yet opened: please encapsulate calls to the reader in a `with` block, or "
+                "explicitly `open` and `close` the resource"};
+        }
+        return iterator{handle_, reader_, batch_size_, table_names_.size()};
+    }
+
+    iterator end() const noexcept
+    {
+        return iterator{};
+    }
+
+private:
+    qdb::logger logger_;
+    qdb::handle_ptr handle_;
+    qdb_reader_handle_t reader_;
+
+    std::vector<std::string> table_names_;
+    std::vector<std::string> column_names_;
+    std::size_t batch_size_;
+    std::vector<py::tuple> ranges_;
+};
+
+using reader_ptr = std::unique_ptr<reader>;
+
+void register_reader(py::module_ & m);
+
+} // namespace qdb

quasardb/table.cpp

@@ -2,9 +2,10 @@
 #include "dispatch.hpp"
 #include "metrics.hpp"
 #include "object_tracker.hpp"
-#include "table_reader.hpp"
+#include "reader.hpp"
 #include "traits.hpp"
 #include "convert/point.hpp"
+#include <memory> // for make_unique
 
 namespace qdb
 {
@@ -100,42 +101,8 @@ void table::_cache_metadata() const
     {
         _ttl = std::chrono::milliseconds{metadata->ttl};
     }
-}
-
-py::object table::reader(
-    const std::vector<std::string> & columns, py::object ranges, bool dict_mode) const
-{
-    _handle->check_open();
-
-    auto ranges_ = qdb::convert_ranges(ranges);
-
-    std::vector<detail::column_info> c_columns;
 
-    if (columns.empty())
-    {
-        // This is a kludge, because technically a table can have no columns, and we're
-        // abusing it as "no argument provided". It's a highly exceptional use case, and
-        // doesn't really have any implication in practice (we just look up twice), so it
-        // should be ok.
-        c_columns = list_columns();
-    }
-    else
-    {
-        c_columns.reserve(columns.size());
-        // This transformation can probably be optimized, but it's only invoked when constructing
-        // the reader so it's unlikely to be a performance bottleneck.
-        std::transform(std::cbegin(columns), std::cend(columns), std::back_inserter(c_columns),
-            [this](const auto & col) {
-                const auto & info = column_info_by_id(col);
-                return detail::column_info{info.type, col, info.symtable};
-            });
-    }
-
-    return (dict_mode == true
-                ? py::cast(qdb::table_reader<reader::ts_dict_row>(_handle, _alias, c_columns, ranges_),
-                    py::return_value_policy::move)
-                : py::cast(qdb::table_reader<reader::ts_fast_row>(_handle, _alias, c_columns, ranges_),
-                    py::return_value_policy::move));
+    _shard_size = std::chrono::milliseconds{metadata->shard_size};
 }
 
 qdb_uint_t table::erase_ranges(const std::string & column, py::object ranges)
@@ -286,4 +253,5 @@ std::pair<pybind11::array, masked_array> table::timestamp_get_ranges(
 
     return ret;
 }
+
 }; // namespace qdb

quasardb/table.hpp

@@ -32,6 +32,7 @@
 
 #include "entry.hpp"
 #include "masked_array.hpp"
+#include "reader.hpp"
 #include "detail/ts_column.hpp"
 
 namespace qdb
@@ -87,7 +88,7 @@ public:
             qdb_ts_insert_columns_ex(*_handle, _alias.c_str(), c_columns.data(), c_columns.size()));
     }
 
-    std::vector<detail::column_info> list_columns() const
+    std::vector<detail::column_info> const & list_columns() const
     {
         if (_columns.has_value()) [[likely]]
         {
@@ -156,8 +157,14 @@ public:
         return column_info_by_id(alias).type;
     }
 
-    py::object reader(
-        const std::vector<std::string> & columns, py::object obj_ranges, bool dict_mode) const;
+    qdb::reader_ptr reader(                            //
+        std::vector<std::string> const & column_names, //
+        std::size_t batch_size,                        //
+        std::vector<py::tuple> const & ranges) const   //
+    {
+        std::vector<std::string> table_names{get_name()};
+        return std::make_unique<qdb::reader>(_handle, table_names, column_names, batch_size, ranges);
+    };
 
     /**
      * Returns true if this table has a TTL assigned.
@@ -196,6 +203,23 @@ public:
         throw qdb::alias_not_found_exception{};
     }
 
+    inline std::chrono::milliseconds get_shard_size() const
+    {
+        if (_shard_size.has_value()) [[likely]]
+        {
+            return _shard_size.value();
+        }
+
+        _cache_metadata();
+
+        if (_shard_size.has_value()) [[likely]]
+        {
+            return _shard_size.value();
+        }
+
+        throw qdb::alias_not_found_exception{};
+    }
+
 private:
     /**
      * Loads column info / metadata from server and caches it locally.
@@ -209,8 +233,9 @@ private:
     {
         if (_columns.has_value() == false) [[unlikely]]
         {
-            // We expect _ttl and _columns to have the same state
+            // We expect _ttl and _columns and _shard_size (i.e. all metadata) to have the same state
             assert(_ttl.has_value() == false);
+            assert(_shard_size.has_value() == false);
             _cache_metadata();
         }
     }
@@ -264,6 +289,7 @@ private:
 
     mutable std::optional<std::vector<detail::column_info>> _columns;
     mutable std::optional<std::chrono::milliseconds> _ttl;
+    mutable std::optional<std::chrono::milliseconds> _shard_size;
 };
 
 template <typename Module>
@@ -285,7 +311,8 @@ static inline void register_table(Module & m)
         .def("__repr__", &qdb::table::repr)                                //
         .def("create", &qdb::table::create, py::arg("columns"),            //
             py::arg("shard_size") = std::chrono::hours{24},                //
-            py::arg("ttl")        = std::chrono::milliseconds::zero())            //
+            py::arg("ttl")        = std::chrono::milliseconds::zero()      //
+            )                                                              //
         .def("get_name", &qdb::table::get_name)                            //
         .def("retrieve_metadata", &qdb::table::retrieve_metadata)          //
         .def("column_index_by_id", &qdb::table::column_index_by_id)        //
@@ -297,29 +324,43 @@ static inline void register_table(Module & m)
         .def("list_columns", &qdb::table::list_columns)                    //
         .def("has_ttl", &qdb::table::has_ttl)                              //
         .def("get_ttl", &qdb::table::get_ttl)                              //
-
-        // We cannot initialize columns with all columns by default, because i don't
-        // see a way to figure out the `this` address for qdb_ts_reader for the default
-        // arguments, and we need it to call qdb::table::list_columns().
-        .def("reader", &qdb::table::reader, py::arg("columns") = std::vector<std::string>(),
-            py::arg("ranges") = py::none{}, py::arg("dict") = false)
-        .def("subscribe", &qdb::table::subscribe)               //
-        .def("erase_ranges", &qdb::table::erase_ranges)         //
-        .def("blob_insert", &qdb::table::blob_insert)           //
-        .def("string_insert", &qdb::table::string_insert)       //
-        .def("double_insert", &qdb::table::double_insert)       //
-        .def("int64_insert", &qdb::table::int64_insert)         //
-        .def("timestamp_insert", &qdb::table::timestamp_insert) //
-        .def("blob_get_ranges", &qdb::table::blob_get_ranges, py::arg("column"),
-            py::arg("ranges") = py::none{}) //
-        .def("string_get_ranges", &qdb::table::string_get_ranges, py::arg("column"),
-            py::arg("ranges") = py::none{}) //
-        .def("double_get_ranges", &qdb::table::double_get_ranges, py::arg("column"),
-            py::arg("ranges") = py::none{}) //
-        .def("int64_get_ranges", &qdb::table::int64_get_ranges, py::arg("column"),
-            py::arg("ranges") = py::none{}) //
-        .def("timestamp_get_ranges", &qdb::table::timestamp_get_ranges, py::arg("column"),
-            py::arg("ranges") = py::none{}); //
+        .def("get_shard_size", &qdb::table::get_shard_size)                //
+                                                                           //
+        .def("reader", &qdb::table::reader,                                //
+            py::kw_only(),                                                 //
+            py::arg("column_names") = std::vector<std::string>{},          //
+            py::arg("batch_size")   = std::size_t{0},                      //
+            py::arg("ranges")       = std::vector<py::tuple>{}             //
+            )                                                              //
+                                                                           //
+        .def("subscribe", &qdb::table::subscribe)                          //
+        .def("erase_ranges", &qdb::table::erase_ranges)                    //
+        .def("blob_insert", &qdb::table::blob_insert)                      //
+        .def("string_insert", &qdb::table::string_insert)                  //
+        .def("double_insert", &qdb::table::double_insert)                  //
+        .def("int64_insert", &qdb::table::int64_insert)                    //
+        .def("timestamp_insert", &qdb::table::timestamp_insert)            //
+                                                                           //
+        .def("blob_get_ranges", &qdb::table::blob_get_ranges,              //
+            py::arg("column"),                                             //
+            py::arg("ranges") = py::none{}                                 //
+            )                                                              //
+        .def("string_get_ranges", &qdb::table::string_get_ranges,          //
+            py::arg("column"),                                             //
+            py::arg("ranges") = py::none{}                                 //
+            )                                                              //
+        .def("double_get_ranges", &qdb::table::double_get_ranges,          //
+            py::arg("column"),                                             //
+            py::arg("ranges") = py::none{}                                 //
+            )                                                              //
+        .def("int64_get_ranges", &qdb::table::int64_get_ranges,            //
+            py::arg("column"),                                             //
+            py::arg("ranges") = py::none{}                                 //
+            )                                                              //
+        .def("timestamp_get_ranges", &qdb::table::timestamp_get_ranges,    //
+            py::arg("column"),                                             //
+            py::arg("ranges") = py::none{}                                 //
+        );                                                                 //
 }
 
 } // namespace qdb

quasardb/traits.hpp

@@ -218,6 +218,21 @@ VALUE_TRAIT_DECL(qdb_blob_t)
     }
 };
 
+VALUE_TRAIT_DECL(char const *)
+{
+    static constexpr bool is_qdb_primitive = false;
+
+    static constexpr char const * null_value() noexcept
+    {
+        return nullptr;
+    }
+
+    static constexpr bool is_null(char const * x) noexcept
+    {
+        return x == null_value();
+    }
+};
+
 VALUE_TRAIT_DECL(qdb_ts_double_point)
 {
     using primitive_type               = double;
@@ -616,4 +631,12 @@ inline typename T::value_type null_value() noexcept
     return T::null_value();
 };
 
+template <class T>
+struct is_chrono_duration : std::false_type
+{};
+
+template <class Rep, class Period>
+struct is_chrono_duration<std::chrono::duration<Rep, Period>> : std::true_type
+{};
+
 }; // namespace qdb::traits