quasardb 3.14.2.dev1__cp310-cp310-win32.whl → 3.14.2.dev2__cp310-cp310-win32.whl

quasardb/CMakeLists.txt

@@ -223,12 +223,6 @@ elseif (MSVC)
 
 
   )
-  if(QDB_CPU_IS_X86)
-    add_compile_options(
-      # Enable AVX as we target Core2 and later
-      /arch:AVX
-    )
-  endif()
 endif()
 
 ###
@@ -357,23 +351,29 @@ set(QDB_FILES
   numpy.cpp
   object_tracker.hpp
   options.hpp
+  properties.cpp
+  properties.hpp
   query.hpp
   query.cpp
+  reader.cpp
+  reader.hpp
   string.hpp
   table.cpp
   table.hpp
-  table_reader.hpp
   tag.hpp
   timestamp.hpp
   traits.hpp
   utils.cpp
   utils.hpp
-  writer.cpp
   writer.hpp
+  detail/invoke.hpp
+  detail/retry.hpp
+  detail/retry.cpp
+  detail/sleep.hpp
   detail/qdb_resource.hpp
   detail/ts_column.hpp
-  reader/ts_row.hpp
-  reader/ts_value.hpp
+  detail/writer.cpp
+  detail/writer.hpp
   utils/blob_deque.hpp
   utils/ostream.hpp
   utils/permutation.hpp
@@ -387,11 +387,18 @@ if(QDB_TESTS_ENABLED)
 ------------------------------------------------------------------ \
 [qdb-api-python] Tests enabled -- do not enable for release builds \
 ------------------------------------------------------------------ \ ")
-  set(QDB_TEST_FILES
 
+  add_compile_definitions(QDB_TESTS_ENABLED)
+
+  set(QDB_TEST_FILES
     ../tests/conftest.cpp
     ../tests/conftest.hpp
-    ../tests/test_convert.cpp)
+    ../tests/test_convert.cpp
+
+    ../tests/detail/mock_failure.hpp
+    ../tests/detail/mock_failure.cpp
+    ../tests/detail/sleep.hpp
+  )
 endif()
 
 # step 3: build

quasardb/INSTALL.vcxproj

@@ -126,6 +126,7 @@ if %errorlevel% neq 0 goto :VCEnd</Command>
   </ItemDefinitionGroup>
   <ItemGroup>
     <CustomBuild Include="C:\TeamCity\work\938b0bdf6727d1ad\build\temp.win32-cpython-310\Release\CMakeFiles\2f5138c6189df16b6f2b038f022b1220\INSTALL_force.rule">
+      <BuildInParallel Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">true</BuildInParallel>
       <Message Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> </Message>
       <Command Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">setlocal
 cd .
@@ -140,6 +141,7 @@ if %errorlevel% neq 0 goto :VCEnd</Command>
       <Outputs Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">C:\TeamCity\work\938b0bdf6727d1ad\build\lib.win32-cpython-310\quasardb\CMakeFiles\INSTALL_force</Outputs>
       <LinkObjects Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</LinkObjects>
       <VerifyInputsAndOutputsExist Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'">false</VerifyInputsAndOutputsExist>
+      <BuildInParallel Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">true</BuildInParallel>
       <Message Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> </Message>
       <Command Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">setlocal
 cd .
@@ -154,6 +156,7 @@ if %errorlevel% neq 0 goto :VCEnd</Command>
       <Outputs Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">C:\TeamCity\work\938b0bdf6727d1ad\build\lib.win32-cpython-310\quasardb\CMakeFiles\INSTALL_force</Outputs>
       <LinkObjects Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</LinkObjects>
       <VerifyInputsAndOutputsExist Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">false</VerifyInputsAndOutputsExist>
+      <BuildInParallel Condition="'$(Configuration)|$(Platform)'=='MinSizeRel|Win32'">true</BuildInParallel>
       <Message Condition="'$(Configuration)|$(Platform)'=='MinSizeRel|Win32'"> </Message>
       <Command Condition="'$(Configuration)|$(Platform)'=='MinSizeRel|Win32'">setlocal
 cd .
@@ -168,6 +171,7 @@ if %errorlevel% neq 0 goto :VCEnd</Command>
       <Outputs Condition="'$(Configuration)|$(Platform)'=='MinSizeRel|Win32'">C:\TeamCity\work\938b0bdf6727d1ad\build\lib.win32-cpython-310\quasardb\CMakeFiles\INSTALL_force</Outputs>
       <LinkObjects Condition="'$(Configuration)|$(Platform)'=='MinSizeRel|Win32'">false</LinkObjects>
       <VerifyInputsAndOutputsExist Condition="'$(Configuration)|$(Platform)'=='MinSizeRel|Win32'">false</VerifyInputsAndOutputsExist>
+      <BuildInParallel Condition="'$(Configuration)|$(Platform)'=='RelWithDebInfo|Win32'">true</BuildInParallel>
       <Message Condition="'$(Configuration)|$(Platform)'=='RelWithDebInfo|Win32'"> </Message>
       <Command Condition="'$(Configuration)|$(Platform)'=='RelWithDebInfo|Win32'">setlocal
 cd .

quasardb/__init__.py

@@ -35,9 +35,19 @@
 .. moduleauthor: quasardb SAS. All rights reserved
 """
 
-def generic_error_msg(msg, e):
+def generic_error_msg(msg, e = None):
     msg_str = "\n".join(msg)
-    return """
+
+    if e is None:
+        return """
+**************************************************************************
+
+{}
+
+**************************************************************************
+""".format(msg_str, type(e), str(e))
+    else:
+        return """
 **************************************************************************
 
 {}
@@ -46,8 +56,8 @@ def generic_error_msg(msg, e):
 
 Original exception:
 
-   Type:    {}
-   Message: {}
+  Type:    {}
+  Message: {}
 
 **************************************************************************
 """.format(msg_str, type(e), str(e))
@@ -79,9 +89,23 @@ def glibc_error_msg(e):
 
     return generic_error_msg(msg, e)
 
+def unknown_error_msg():
+    msg = [
+        "Unable to import quasardb module: unknown error. ",
+        "",
+        "This may have several causes, including, but not limited to: ",
+        " - a link error, ",
+        " - architecture mismatch, ",
+        " - the current working directory already contains a 'quasardb' subdirectory.",
+        "",
+        "If you believe this to be a bug, please reach out to QuasarDB at",
+        "support@quasar.ai"]
+
+    return generic_error_msg(msg)
 try:
     from quasardb.quasardb import *
 except BaseException as e:
+    print(e)
     if "undefined symbol" in str(e):
         print(link_error_msg(e))
         raise e
@@ -90,5 +114,10 @@ except BaseException as e:
     else:
         from quasardb import *
 
+if not 'quasardb' in locals():
+    print(unknown_error_msg())
+    raise ImportError()
+
+
 from .extensions import extend_module
 extend_module(quasardb)

quasardb/cluster.cpp

@@ -13,7 +13,8 @@ cluster::cluster(const std::string & uri,
     const std::string & user_security_file,
     const std::string & cluster_public_key_file,
     std::chrono::milliseconds timeout,
-    bool do_version_check)
+    bool do_version_check,
+    std::size_t client_max_parallelism)
     : _uri{uri}
     , _handle{make_handle_ptr()}
     , _json_loads{pybind11::module::import("json").attr("loads")}
@@ -31,6 +32,11 @@ cluster::cluster(const std::string & uri,
 
     options().set_timeout(timeout);
 
+    if (client_max_parallelism != 0)
+    {
+        options().set_client_max_parallelism(client_max_parallelism);
+    }
+
     // HACKS(leon): we need to ensure there is always one callback active
     //              for qdb. Callbacks can be lost when the last active session
     //              gets closed. As such, the most pragmatic place to 'check'

quasardb/cluster.hpp

@@ -41,10 +41,11 @@
 #include "node.hpp"
 #include "options.hpp"
 #include "perf.hpp"
+#include "properties.hpp"
 #include "query.hpp"
+#include "reader.hpp"
 #include "string.hpp"
 #include "table.hpp"
-#include "table_reader.hpp"
 #include "tag.hpp"
 #include "timestamp.hpp"
 #include "utils.hpp"
@@ -72,7 +73,8 @@ public:
         const std::string & user_security_file      = {},
         const std::string & cluster_public_key_file = {},
         std::chrono::milliseconds timeout           = std::chrono::minutes{1},
-        bool do_version_check                       = false);
+        bool do_version_check                       = false,
+        std::size_t client_max_parallelism          = 0);
 
 public:
     void close();
@@ -239,6 +241,18 @@ public:
         return qdb::table{_handle, alias};
     }
 
+    // the reader_ptr is non-copyable
+    qdb::reader_ptr reader(                            //
+        std::vector<std::string> const & table_names,  //
+        std::vector<std::string> const & column_names, //
+        std::size_t batch_size,                        //
+        std::vector<py::tuple> const & ranges)         //
+    {
+        check_open();
+
+        return std::make_unique<qdb::reader>(_handle, table_names, column_names, batch_size, ranges);
+    }
+
     // the batch_inserter_ptr is non-copyable
     qdb::batch_inserter_ptr inserter(const std::vector<batch_column_info> & ci)
     {
@@ -270,6 +284,13 @@ public:
         return qdb::options{_handle};
     }
 
+    qdb::properties properties()
+    {
+        check_open();
+
+        return qdb::properties{_handle};
+    }
+
     qdb::perf perf()
     {
         check_open();
@@ -334,22 +355,30 @@ public:
         return py::cast(qdb::numpy_query(_handle, query_string));
     }
 
-    std::shared_ptr<qdb::query_continuous> query_continuous_full(
-        const std::string & query_string, std::chrono::milliseconds pace, const py::object & blobs)
+    std::shared_ptr<qdb::query_continuous> query_continuous(qdb_query_continuous_mode_type_t mode,
+        const std::string & query_string,
+        std::chrono::milliseconds pace,
+        const py::object & blobs)
     {
         check_open();
 
-        return std::make_shared<qdb::query_continuous>(
-            _handle, qdb_query_continuous_full, pace, query_string, blobs);
+        auto o = std::make_shared<qdb::query_continuous>(_handle, blobs);
+
+        o->run(mode, pace, query_string);
+
+        return o;
     }
 
-    std::shared_ptr<qdb::query_continuous> query_continuous_new_values(
+    std::shared_ptr<qdb::query_continuous> query_continuous_full(
         const std::string & query_string, std::chrono::milliseconds pace, const py::object & blobs)
     {
-        check_open();
+        return query_continuous(qdb_query_continuous_full, query_string, pace, blobs);
+    }
 
-        return std::make_shared<qdb::query_continuous>(
-            _handle, qdb_query_continuous_new_values_only, pace, query_string, blobs);
+    std::shared_ptr<qdb::query_continuous> query_continuous_new_values(
+        const std::string & query_string, std::chrono::milliseconds pace, const py::object & blobs)
+    {
+        return query_continuous(qdb_query_continuous_new_values_only, query_string, pace, blobs);
     }
 
 public:
@@ -484,68 +513,78 @@ static inline void register_cluster(Module & m)
 
     py::class_<qdb::cluster>(m, "Cluster",
         "Represents a connection to the QuasarDB cluster. ") //
-        .def(
-            py::init<const std::string &, const std::string &, const std::string &, const std::string &,
-                const std::string &, const std::string &, std::chrono::milliseconds, bool>(), //
-            py::arg("uri"),                                                                   //
-            py::arg("user_name")               = std::string{},                               //
-            py::arg("user_private_key")        = std::string{},                               //
-            py::arg("cluster_public_key")      = std::string{},                               //
-            py::arg("user_security_file")      = std::string{},                               //
-            py::arg("cluster_public_key_file") = std::string{},                               //
-            py::arg("timeout")                 = std::chrono::minutes{1},                     //
-            py::arg("do_version_check")        = false)                                              //
-        .def("__enter__", &qdb::cluster::enter)                                               //
-        .def("__exit__", &qdb::cluster::exit)                                                 //
-        .def("tidy_memory", &qdb::cluster::tidy_memory)                                       //
-        .def("get_memory_info", &qdb::cluster::get_memory_info)                               //
-        .def("is_open", &qdb::cluster::is_open)                                               //
-        .def("uri", &qdb::cluster::uri)                                                       //
-        .def("node", &qdb::cluster::node)                                                     //
-        .def("options", &qdb::cluster::options)                                               //
-        .def("perf", &qdb::cluster::perf)                                                     //
-        .def("node_status", &qdb::cluster::node_status)                                       //
-        .def("node_config", &qdb::cluster::node_config)                                       //
-        .def("node_topology", &qdb::cluster::node_topology)                                   //
-        .def("tag", &qdb::cluster::tag)                                                       //
-        .def("blob", &qdb::cluster::blob)                                                     //
-        .def("string", &qdb::cluster::string)                                                 //
-        .def("integer", &qdb::cluster::integer)                                               //
-        .def("double", &qdb::cluster::double_)                                                //
-        .def("timestamp", &qdb::cluster::timestamp)                                           //
-        .def("ts", &qdb::cluster::table)                                                      //
-        .def("table", &qdb::cluster::table)                                                   //
-        .def("ts_batch", &qdb::cluster::inserter)                                             //
-        .def("inserter", &qdb::cluster::inserter)                                             //
-        .def("pinned_writer", &qdb::cluster::pinned_writer)                                   //
-        .def("writer", &qdb::cluster::writer)                                                 //
-        .def("find", &qdb::cluster::find)                                                     //
-        .def("query", &qdb::cluster::query,                                                   //
-            py::arg("query"),                                                                 //
-            py::arg("blobs") = false)                                                         //
-        .def("query_numpy", &qdb::cluster::query_numpy,                                       //
-            py::arg("query"))                                                                 //
-        .def("query_continuous_full", &qdb::cluster::query_continuous_full,                   //
-            py::arg("query"),                                                                 //
-            py::arg("pace"),                                                                  //
-            py::arg("blobs") = false)                                                         //
-        .def("query_continuous_new_values", &qdb::cluster::query_continuous_new_values,       //
-            py::arg("query"),                                                                 //
-            py::arg("pace"),                                                                  //
-            py::arg("blobs") = false)                                                         //
-        .def("prefix_get", &qdb::cluster::prefix_get)                                         //
-        .def("prefix_count", &qdb::cluster::prefix_count)                                     //
-        .def("suffix_get", &qdb::cluster::suffix_get)                                         //
-        .def("suffix_count", &qdb::cluster::suffix_count)                                     //
-        .def("close", &qdb::cluster::close)                                                   //
-        .def("purge_all", &qdb::cluster::purge_all)                                           //
-        .def("trim_all", &qdb::cluster::trim_all)                                             //
-        .def("purge_cache", &qdb::cluster::purge_cache)                                       //
-        .def("compact_full", &qdb::cluster::compact_full)                                     //
-        .def("compact_progress", &qdb::cluster::compact_progress)                             //
-        .def("compact_abort", &qdb::cluster::compact_abort)                                   //
-        .def("wait_for_compaction", &qdb::cluster::wait_for_compaction)                       //
-        .def("endpoints", &qdb::cluster::endpoints);                                          //
+        .def(py::init<const std::string &, const std::string &, const std::string &,
+                 const std::string &, const std::string &, const std::string &,
+                 std::chrono::milliseconds, bool, std::size_t>(),                       //
+            py::arg("uri"),                                                             //
+            py::arg("user_name")               = std::string{},                         //
+            py::arg("user_private_key")        = std::string{},                         //
+            py::arg("cluster_public_key")      = std::string{},                         //
+            py::arg("user_security_file")      = std::string{},                         //
+            py::arg("cluster_public_key_file") = std::string{},                         //
+            py::arg("timeout")                 = std::chrono::minutes{1},               //
+            py::arg("do_version_check")        = false,                                 //
+            py::arg("client_max_parallelism")  = std::size_t{0}                         //
+            )                                                                           //
+        .def("__enter__", &qdb::cluster::enter)                                         //
+        .def("__exit__", &qdb::cluster::exit)                                           //
+        .def("tidy_memory", &qdb::cluster::tidy_memory)                                 //
+        .def("get_memory_info", &qdb::cluster::get_memory_info)                         //
+        .def("is_open", &qdb::cluster::is_open)                                         //
+        .def("uri", &qdb::cluster::uri)                                                 //
+        .def("node", &qdb::cluster::node)                                               //
+        .def("options", &qdb::cluster::options)                                         //
+        .def("properties", &qdb::cluster::properties)                                   //
+        .def("perf", &qdb::cluster::perf)                                               //
+        .def("node_status", &qdb::cluster::node_status)                                 //
+        .def("node_config", &qdb::cluster::node_config)                                 //
+        .def("node_topology", &qdb::cluster::node_topology)                             //
+        .def("tag", &qdb::cluster::tag)                                                 //
+        .def("blob", &qdb::cluster::blob)                                               //
+        .def("string", &qdb::cluster::string)                                           //
+        .def("integer", &qdb::cluster::integer)                                         //
+        .def("double", &qdb::cluster::double_)                                          //
+        .def("timestamp", &qdb::cluster::timestamp)                                     //
+        .def("ts", &qdb::cluster::table)                                                //
+        .def("table", &qdb::cluster::table)                                             //
+        .def("ts_batch", &qdb::cluster::inserter)                                       //
+        .def("inserter", &qdb::cluster::inserter)                                       //
+        .def("reader", &qdb::cluster::reader,                                           //
+            py::arg("table_names"),                                                     //
+            py::kw_only(),                                                              //
+            py::arg("column_names") = std::vector<std::string>{},                       //
+            py::arg("batch_size")   = std::size_t{0},                                   //
+            py::arg("ranges")       = std::vector<py::tuple>{}                          //
+            )                                                                           //
+        .def("pinned_writer", &qdb::cluster::pinned_writer)                             //
+        .def("writer", &qdb::cluster::writer)                                           //
+        .def("find", &qdb::cluster::find)                                               //
+        .def("query", &qdb::cluster::query,                                             //
+            py::arg("query"),                                                           //
+            py::arg("blobs") = false)                                                   //
+        .def("query_numpy", &qdb::cluster::query_numpy,                                 //
+            py::arg("query"))                                                           //
+        .def("query_continuous_full", &qdb::cluster::query_continuous_full,             //
+            py::arg("query"),                                                           //
+            py::arg("pace"),                                                            //
+            py::arg("blobs") = false)                                                   //
+        .def("query_continuous_new_values", &qdb::cluster::query_continuous_new_values, //
+            py::arg("query"),                                                           //
+            py::arg("pace"),                                                            //
+            py::arg("blobs") = false)                                                   //
+        .def("prefix_get", &qdb::cluster::prefix_get)                                   //
+        .def("prefix_count", &qdb::cluster::prefix_count)                               //
+        .def("suffix_get", &qdb::cluster::suffix_get)                                   //
+        .def("suffix_count", &qdb::cluster::suffix_count)                               //
+        .def("close", &qdb::cluster::close)                                             //
+        .def("purge_all", &qdb::cluster::purge_all)                                     //
+        .def("trim_all", &qdb::cluster::trim_all)                                       //
+        .def("purge_cache", &qdb::cluster::purge_cache)                                 //
+        .def("compact_full", &qdb::cluster::compact_full)                               //
+        .def("compact_progress", &qdb::cluster::compact_progress)                       //
+        .def("compact_abort", &qdb::cluster::compact_abort)                             //
+        .def("wait_for_compaction", &qdb::cluster::wait_for_compaction)                 //
+        .def("endpoints", &qdb::cluster::endpoints);                                    //
 }
 
 } // namespace qdb

quasardb/concepts.hpp

@@ -4,6 +4,7 @@
 #include <pybind11/numpy.h>
 #include <range/v3/range/concepts.hpp>
 #include <range/v3/range/traits.hpp>
+#include <chrono>
 #include <iterator>
 #include <type_traits>
 
@@ -114,12 +115,13 @@ static_assert(not qdb_primitive<qdb_ts_string_point>);
 template <typename Dtype>
 concept dtype = std::is_base_of_v<traits::dtype<Dtype::kind>, Dtype>
 
-    && std::is_enum_v<decltype(Dtype::kind)>
+                && std::is_enum_v<decltype(Dtype::kind)>
 
     ;
 
 template <typename Dtype>
-concept fixed_width_dtype = dtype<Dtype>
+concept fixed_width_dtype =
+    dtype<Dtype>
 
     // Check base class
     && std::is_base_of_v<traits::fixed_width_dtype<Dtype::kind, Dtype::size>, Dtype>;
@@ -147,14 +149,14 @@ concept dtype8 = dtype_of_width<Dtype, 1>;
 template <typename Dtype>
 concept datetime64_dtype = fixed_width_dtype<Dtype>
 
-    // Verify base class is datetime64_dtype
-    && std::is_base_of_v<traits::datetime64_dtype<Dtype::precision>, Dtype>
+                           // Verify base class is datetime64_dtype
+                           && std::is_base_of_v<traits::datetime64_dtype<Dtype::precision>, Dtype>
 
-    // datetime64 is always a 64bit object
-    && dtype64<Dtype>
+                           // datetime64 is always a 64bit object
+                           && dtype64<Dtype>
 
-    // It needs to have a precision
-    && std::is_enum_v<decltype(Dtype::precision)>;
+                           // It needs to have a precision
+                           && std::is_enum_v<decltype(Dtype::precision)>;
 
 template <typename Dtype>
 concept variable_width_dtype =
@@ -173,11 +175,11 @@ concept variable_width_dtype =
 template <typename Dtype>
 concept object_dtype = dtype<Dtype>
 
-    // Objects are always fixed width (64-bit pointers, effectively)
-    && fixed_width_dtype<Dtype>
+                       // Objects are always fixed width (64-bit pointers, effectively)
+                       && fixed_width_dtype<Dtype>
 
-    // Verify base class
-    && std::is_base_of_v<traits::object_dtype<typename Dtype::value_type>, Dtype>;
+                       // Verify base class
+                       && std::is_base_of_v<traits::object_dtype<typename Dtype::value_type>, Dtype>;
 
 // Trivial dtypes are useful for deciding whether you can use fast memcpy-based
 // conversions, e.g. when numpy's int64 has the exact same representation as
@@ -229,6 +231,47 @@ concept delegate_dtype =
     // very least nothrow_convertible.
     && std::is_nothrow_convertible_v<delegate_from_type_t<Dtype>, delegate_to_type_t<Dtype>>;
 
+////////////////////////////////////////////////////////////////////////////////
+//
+// Python API useful concepts, for e.g. batch writer
+//
+///////////////////
+
+template <typename T>
+concept duration = traits::is_chrono_duration<T>::value;
+
+template <typename T, typename Duration = T::duration_t>
+concept sleep_strategy =
+    // Should always be a valid chrono duration
+    duration<Duration>
+
+    // And have a sleep function that takes a duration, and returns void
+    && requires(T s, Duration duration_) {
+           {
+               s.sleep(duration_)
+           } -> std::same_as<void>;
+       };
+
+template <typename T>
+concept writer_push_strategy =
+    // Check that we can do a batch push invoke
+    requires(                                                     //
+        T t,                                                      //
+        qdb_handle_t handle,                                      //
+        qdb_exp_batch_options_t const * options,                  //
+        qdb_exp_batch_push_table_t const * tables,                //
+        qdb_exp_batch_push_table_schema_t const ** table_schemas, //
+        qdb_size_t table_count,                                   //
+        py::kwargs const & kwargs                                 //
+    ) {
+        {
+            t(handle, options, tables, table_schemas, table_count)
+        } -> std::same_as<qdb_error_t>;
+        {
+            T::from_kwargs(kwargs)
+        } -> std::same_as<T>;
+    };
+
 // Assertions
 static_assert(dtype<traits::unicode_dtype>);
 
@@ -275,4 +318,5 @@ static_assert(not trivial_dtype<traits::float32_dtype>);
 
 static_assert(fixed_width_dtype<traits::float64_dtype>);
 static_assert(fixed_width_dtype<traits::float32_dtype>);
+
 }; // namespace qdb::concepts