quasardb 3.14.2.dev1__cp311-cp311-macosx_11_0_arm64.whl → 3.14.2.dev3__cp311-cp311-macosx_11_0_arm64.whl

quasardb/module.cpp

@@ -2,12 +2,26 @@
 #include "cluster.hpp"
 #include "metrics.hpp"
 #include "node.hpp"
+#include "reader.hpp"
 #include "writer.hpp"
 #include <functional>
 #include <list>
 
+#ifndef QDB_TESTS_ENABLED
+#    include "detail/sleep.hpp"
+#    include "detail/writer.hpp"
+using push_strategy_t  = qdb::detail::default_writer_push_strategy;
+using sleep_strategy_t = qdb::detail::default_sleep_strategy<>;
+#else
+#    include "../tests/detail/sleep.hpp"
+#    include "../tests/detail/writer.hpp"
+using push_strategy_t  = qdb::detail::mock_failure_writer_push_strategy;
+using sleep_strategy_t = qdb::detail::mock_sleep_strategy<>;
+#endif
+
 namespace qdb
 {
+
 /**
  * This approach is inspired from pybind11, where we keep a global static variable
  * of initializers for components/modules that which to register with our global
@@ -45,6 +59,7 @@ PYBIND11_MODULE(quasardb, m)
     qdb::register_cluster(m);
     qdb::register_node(m);
     qdb::register_options(m);
+    qdb::register_properties(m);
     qdb::register_perf(m);
     qdb::register_entry(m);
     qdb::register_double(m);
@@ -60,14 +75,15 @@ PYBIND11_MODULE(quasardb, m)
     qdb::register_table(m);
     qdb::register_batch_column(m);
     qdb::register_batch_inserter(m);
-    qdb::register_table_reader(m);
     qdb::register_masked_array(m);
-    qdb::register_writer(m);
+    qdb::register_reader(m);
+
+    qdb::register_writer<push_strategy_t, sleep_strategy_t>(m);
+
     qdb::register_metrics(m);
 
     qdb::detail::register_ts_column(m);
-    qdb::reader::register_ts_value(m);
-    qdb::reader::register_ts_row(m);
+    qdb::detail::register_retry_options(m);
 
     for (const auto & initializer : qdb::initializers())
     {

quasardb/node.hpp

@@ -51,13 +51,19 @@ public:
         const std::string & user_private_key        = {},
         const std::string & cluster_public_key      = {},
         const std::string & user_security_file      = {},
-        const std::string & cluster_public_key_file = {})
+        const std::string & cluster_public_key_file = {},
+        bool enable_encryption = false)
         : _uri{node_uri}
         , _handle{make_handle_ptr()}
         , _direct_handle{make_direct_handle_ptr()}
     {
         qdb::options{_handle}.apply_credentials(user_name, user_private_key, cluster_public_key,
             user_security_file, cluster_public_key_file);
+
+        if (enable_encryption) {
+          qdb::options{_handle}.set_encryption(qdb_crypt_aes_gcm_256);
+        }
+        
         _direct_handle->connect(_handle, node_uri);
     }
 
@@ -108,13 +114,16 @@ static inline void register_node(Module & m)
 
     py::class_<qdb::node>(m, "Node")
         .def(py::init<std::string const &, std::string const &, std::string const &, std::string const &,
-                std::string const &, std::string const &>(),
-            py::arg("uri"),                                     //
-            py::arg("user_name")          = std::string{},      //
-            py::arg("user_private_key")   = std::string{},      //
-            py::arg("cluster_public_key") = std::string{},      //
-            py::arg("user_security_file")      = std::string{}, //
-            py::arg("cluster_public_key_file") = std::string{}) //
+             std::string const &, std::string const &, bool>(),
+            py::arg("uri"),                                      //
+            py::arg("user_name")               = std::string{},  //
+            py::arg("user_private_key")        = std::string{},  //
+            py::arg("cluster_public_key")      = std::string{},  //
+            py::kw_only(),                                       //             
+            py::arg("user_security_file")      = std::string{},  //
+            py::arg("cluster_public_key_file") = std::string{},  //
+            py::arg("enable_encryption")       = false           //
+             ) //
         .def("prefix_get", &qdb::node::prefix_get)
         .def("blob", &qdb::node::blob)
         .def("integer", &qdb::node::integer);

quasardb/numpy/__init__.py

@@ -414,6 +414,17 @@ def _ensure_list(xs, cinfos):
     return ret
 
 
+def _coerce_retries(retries) -> quasardb.RetryOptions:
+    if retries is None:
+        return quasardb.RetryOptions()
+    elif isinstance(retries, int):
+        return quasardb.RetryOptions(retries=retries)
+    elif isinstance(retries, quasardb.RetryOptions):
+        return retries
+    else:
+        raise TypeError("retries should either be an integer or quasardb.RetryOptions, got: " + type(retries))
+
+
 def ensure_ma(xs, dtype=None):
     if isinstance(dtype, list):
         assert(isinstance(xs, list) == True)
@@ -556,20 +567,25 @@ def write_array(
     logger.info("Writing array (%d rows of dtype %s) to columns %s.%s (type %s)", len(data), data.dtype, table.get_name(), column, ctype)
     write_with[ctype](column, index, data)
 
-
 def write_arrays(
         data,
         cluster,
         table = None,
+        *,
         dtype = None,
         index = None,
         _async = False,
         fast = False,
         truncate = False,
-        deduplicate=False,
-        deduplication_mode='drop',
+        deduplicate = False,
+        deduplication_mode = 'drop',
         infer_types = True,
-        writer = None):
+        writer = None,
+        write_through = False,
+        retries = 3,
+
+        # We accept additional kwargs that will be passed through the writer.push() methods
+        **kwargs):
     """
     Write multiple aligned numpy arrays to a table.
 
@@ -652,12 +668,25 @@ def write_arrays(
 
       Defaults to False.
 
+    write_through: optional bool
+      If True, data is not cached after write.
+      By default is False, in which case caching is left at the discretion of the server.
+
     writer: optional quasardb.Writer
       Allows you to explicitly provide a Writer to use, which is expected to be
       initialized with the `table`.
 
       Reuse of the Writer allows for some performance improvements.
 
+    retries: optional int or quasardb.RetryOptions
+      Number of times to retry in case of a push failure. This is useful in case of async
+      pipeline failures, or when doing transactional inserts that may occasionally cause
+      transaction conflicts.
+
+      Retries with exponential backoff, starts at 3 seconds, and doubles every retry attempt.
+
+      Alternatively, a quasardb.RetryOptions object can be passed to more carefully fine-tune
+      retry behavior.
     """
 
     if table:
@@ -673,7 +702,10 @@ def write_arrays(
                             deduplicate=deduplicate,
                             deduplication_mode=deduplication_mode,
                             infer_types=infer_types,
-                            writer=writer)
+                            write_through=write_through,
+                            writer=writer,
+                            retries=retries,
+                            **kwargs)
 
 
     ret = []
@@ -738,19 +770,35 @@ def write_arrays(
         n_rows += len(index_)
         ret.append(table)
 
+    retries = _coerce_retries(retries)
+
+    # By default, we push all additional kwargs to the writer.push() function. This allows transparent propagation
+    # arguments.
+    #
+    # The initial use case was that so we can add additional parameters for test mocks, e.g. `mock_failures` so that
+    # we can validate the retry functionality.
+    push_kwargs = kwargs
+    push_kwargs['deduplicate'] = deduplicate
+    push_kwargs['deduplication_mode'] = deduplication_mode
+    push_kwargs['write_through'] = write_through
+    push_kwargs['retries'] = retries
+
     logger.debug("pushing %d rows", n_rows)
     start = time.time()
 
     if fast is True:
-        writer.push_fast(push_data, deduplicate=deduplicate, deduplication_mode=deduplication_mode)
+        push_kwargs['push_mode'] = quasardb.WriterPushMode.Fast
     elif truncate is True:
-        writer.push_truncate(push_data, deduplicate=deduplicate, deduplication_mode=deduplication_mode)
+        push_kwargs['push_mode'] = quasardb.WriterPushMode.Truncate
     elif isinstance(truncate, tuple):
-        writer.push_truncate(push_data, range=truncate, deduplicate=deduplicate, deduplication_mode=deduplication_mode)
+        push_kwargs['push_mode'] = quasardb.WriterPushMode.Truncate
+        push_kwargs['range'] = truncate
     elif _async is True:
-        writer.push_async(push_data, deduplicate=deduplicate, deduplication_mode=deduplication_mode)
+        push_kwargs['push_mode'] = quasardb.WriterPushMode.Async
     else:
-        writer.push(push_data, deduplicate=deduplicate, deduplication_mode=deduplication_mode)
+        push_kwargs['push_mode'] = quasardb.WriterPushMode.Transactional
+
+    writer.push(push_data, **push_kwargs)
 
     logger.debug("pushed %d rows in %s seconds",
                 n_rows, (time.time() - start))

quasardb/object_tracker.hpp

@@ -92,8 +92,7 @@ private:
     using container_t = std::vector<trackable_object>;
 
 public:
-    repository()  = default;
-    ~repository() = default;
+    repository() = default;
 
     repository(repository const &)             = delete;
     repository(repository && other)            = delete;
@@ -135,7 +134,7 @@ public:
     };
 
     template <typename T>
-    requires(!std::is_pointer_v<T>) [[nodiscard]] inline T * track(T * x) noexcept
+    [[nodiscard]] inline T * track(T * x) noexcept
     {
         return static_cast<T *>(track(make_trackable_object(std::unique_ptr<T>{x})));
     };

quasardb/options.hpp

@@ -32,6 +32,7 @@
 
 #include "handle.hpp"
 #include <qdb/option.h>
+#include "detail/qdb_resource.hpp"
 #include <chrono>
 
 namespace qdb
@@ -81,6 +82,21 @@ public:
         }
     };
 
+    void set_timezone(std::string const & tz)
+    {
+        qdb::qdb_throw_if_error(*_handle, qdb_option_set_timezone(*_handle, tz.c_str()));
+    }
+
+    std::string get_timezone()
+    {
+        detail::qdb_resource<char const> tz{*_handle};
+        qdb::qdb_throw_if_error(*_handle, qdb_option_get_timezone(*_handle, &tz));
+
+        std::string ret{tz.get()};
+
+        return ret;
+    }
+
     void set_timeout(std::chrono::milliseconds ms)
     {
         qdb::qdb_throw_if_error(
@@ -96,6 +112,16 @@ public:
         return std::chrono::milliseconds{ms};
     }
 
+    void enable_user_properties()
+    {
+        qdb::qdb_throw_if_error(*_handle, qdb_option_enable_user_properties(*_handle));
+    }
+
+    void disable_user_properties()
+    {
+        qdb::qdb_throw_if_error(*_handle, qdb_option_disable_user_properties(*_handle));
+    }
+
     void set_client_soft_memory_limit(std::size_t limit)
     {
         qdb::qdb_throw_if_error(*_handle,
@@ -211,7 +237,8 @@ static inline void register_options(Module & m)
     py::enum_<qdb_compression_t>{o, "Compression", py::arithmetic(), "Compression type"} //
         .value("Disabled", qdb_comp_none)                                                //
         .value("Fast", qdb_comp_fast)                                                    //
-        .value("Best", qdb_comp_best);                                                   //
+        .value("Best", qdb_comp_best)                                                    //
+        .value("Balanced", qdb_comp_balanced);                                           //
 
     py::enum_<qdb_encryption_t>{o, "Encryption", py::arithmetic(), "Encryption type"} //
         .value("Disabled", qdb_crypt_none)                                            //
@@ -220,6 +247,10 @@ static inline void register_options(Module & m)
     o.def(py::init<qdb::handle_ptr>())                                                    //
         .def("set_timeout", &qdb::options::set_timeout)                                   //
         .def("get_timeout", &qdb::options::get_timeout)                                   //
+        .def("set_timezone", &qdb::options::set_timezone)                                 //
+        .def("get_timezone", &qdb::options::get_timezone)                                 //
+        .def("enable_user_properties", &qdb::options::enable_user_properties)             //
+        .def("disable_user_properties", &qdb::options::disable_user_properties)           //
         .def("set_stabilization_max_wait", &qdb::options::set_stabilization_max_wait)     //
         .def("get_stabilization_max_wait", &qdb::options::get_stabilization_max_wait)     //
         .def("set_max_cardinality", &qdb::options::set_max_cardinality)                   //
@@ -230,8 +261,6 @@ static inline void register_options(Module & m)
         .def("set_client_max_in_buf_size", &qdb::options::set_client_max_in_buf_size)     //
         .def("get_client_max_in_buf_size", &qdb::options::get_client_max_in_buf_size)     //
         .def("get_cluster_max_in_buf_size", &qdb::options::get_cluster_max_in_buf_size)   //
-        .def("set_client_max_parallelism", &qdb::options::set_client_max_parallelism,     //
-            py::arg("parallelism"))                                                       //
         .def("get_client_max_parallelism", &qdb::options::get_client_max_parallelism)     //
         .def("set_query_max_length", &qdb::options::set_query_max_length,                 //
             py::arg("query_max_length"))                                                  //

quasardb/pandas/__init__.py

@@ -205,9 +205,10 @@ def query(cluster: quasardb.Cluster,
     df.set_index(index, inplace=True)
     return df
 
-def read_dataframe(table, row_index=False, columns=None, ranges=None):
+def stream_dataframe(table : quasardb.Table, *, batch_size : int = 0, column_names : list = None, ranges : list = None):
     """
-    Read a Pandas Dataframe from a QuasarDB Timeseries table.
+    Read a Pandas Dataframe from a QuasarDB Timeseries table. Returns a generator with dataframes of size `batch_size`, which is useful
+    when traversing a large dataset which does not fit into memory.
 
     Parameters:
     -----------
@@ -215,52 +216,65 @@ def read_dataframe(table, row_index=False, columns=None, ranges=None):
     table : quasardb.Timeseries
       QuasarDB Timeseries table object, e.g. qdb_cluster.table('my_table')
 
-    columns : optional list
+    batch_size : int
+      The amount of rows to fetch in a single read operation. If unset, or 0, will
+      return all data as an entire dataframe. Otherwise, returns a generator which
+      yields on dataframe at a time.
+
+    column_names : optional list
       List of columns to read in dataframe. The timestamp column '$timestamp' is
       always read.
 
       Defaults to all columns.
 
-    row_index: boolean
-      Whether or not to index by rows rather than timestamps. Set to true if your
-      dataset may contains null values and multiple rows may use the same timestamps.
-      Note: using row_index is significantly slower.
-      Defaults to false.
-
     ranges: optional list
       A list of time ranges to read, represented as tuples of Numpy datetime64[ns] objects.
       Defaults to the entire table.
 
     """
+    # Sanitize batch_size
+    if batch_size == None:
+        batch_size = 0
+    elif not isinstance(batch_size, int):
+        raise TypeError("batch_size should be an integer, but got: {} with value {}".format(type(batch_size), str(batch_size)))
 
-    if columns is None:
-        columns = list(c.name for c in table.list_columns())
+    kwargs = {}
 
-    if not row_index:
-        xs = dict((c, (read_series(table, c, ranges))) for c in columns)
-        return DataFrame(data=xs)
+    if column_names:
+        kwargs['column_names'] = column_names
 
-    kwargs = {
-        'columns': columns
-    }
     if ranges:
         kwargs['ranges'] = ranges
 
-    logger.debug("reading DataFrame from bulk reader")
+    with table.reader(**kwargs) as reader:
+        for batch in reader:
+            # We always expect the timestamp column, and set this as the index
+            assert '$timestamp' in batch
+
+            idx = pd.Index(batch.pop('$timestamp'), copy=False, name='$timestamp')
+            df = pd.DataFrame(batch, index=idx)
+
+            yield df
+
 
-    reader = table.reader(**kwargs)
-    xs = []
-    for row in reader:
-        xs.append(row.copy())
+def read_dataframe(table, **kwargs):
+    """
+    Read a Pandas Dataframe from a QuasarDB Timeseries table. Wraps around stream_dataframe(), and
+    returns everything as a single dataframe. batch_size is always explicitly set to 0.
+    """
 
-    columns.insert(0, '$timestamp')
+    if 'batch_size' in kwargs and kwargs['batch_size'] != 0 and kwargs['batch_size'] != None:
+        logger.warn("Providing a batch size with read_dataframe is unsupported, overriding batch_size to 0.")
+        logger.warn("If you wish to traverse the data in smaller batches, please use: stream_dataframe().")
+        kwargs['batch_size'] = 0
 
-    logger.debug(
-        "read %d rows, returning as DataFrame with %d columns",
-        len(xs),
-        len(columns))
+    # Note that this is *lazy*, dfs is a generator, not a list -- as such, dataframes will be
+    # fetched on-demand, which means that an error could occur in the middle of processing
+    # dataframes.
+    dfs = stream_dataframe(table, **kwargs)
+
+    return pd.concat(dfs)
 
-    return DataFrame(data=xs, columns=columns)
 
 def _extract_columns(df, cinfos):
     """
@@ -288,88 +302,39 @@ def _extract_columns(df, cinfos):
 def write_dataframes(
         dfs,
         cluster,
-        dtype=None,
-        create=False,
-        shard_size=None,
-        _async=False,
-        fast=False,
-        truncate=False,
-        deduplicate=False,
-        deduplication_mode='drop',
-        infer_types=True,
-        writer=None):
+        *,
+        create = False,
+        shard_size = None,
+        **kwargs):
     """
-    Store a dataframe into a table with the pin column API.
+    Store dataframes into a table. Any additional parameters not documented here
+    are passed to numpy.write_arrays(). Please consult the pydoc of that function
+    for additional accepted parameters.
 
     Parameters:
     -----------
 
-    df: pandas.DataFrame
-      The pandas dataframe to store.
+    dfs: dict[str | quasardb.Table, pd.DataFrame] | list[tuple[str | quasardb.Table, pd.DataFrame]]
+      This can be either a dict that maps table (either objects or names) to a dataframe, or a list
+      of table<>dataframe tuples.
 
     cluster: quasardb.Cluster
       Active connection to the QuasarDB cluster
 
-    table: quasardb.Timeseries or str
-      Either a string or a reference to a QuasarDB Timeseries table object.
-      For example, 'my_table' or cluster.table('my_table') are both valid values.
-
-    dtype: optional dtype, list of dtype, or dict of dtype
-      Optional data type to force. If a single dtype, will force that dtype to all
-      columns. If list-like, will map dtypes to dataframe columns by their offset.
-      If dict-like, will map dtypes to dataframe columns by their label.
-
-      If a dtype for a column is provided in this argument, and infer_types is also
-      True, this argument takes precedence.
-
     create: optional bool
-      Whether to create the table. Defaults to false.
+      Whether to create the table. Defaults to False.
 
     shard_size: optional datetime.timedelta
-      The shard size of the timeseries you wish to create.
-
-    deduplicate: bool or list[str]
-      Enables server-side deduplication of data when it is written into the table.
-      When True, automatically deduplicates rows when all values of a row are identical.
-      When a list of strings is provided, deduplicates only based on the values of
-      these columns.
-
-      Defaults to False.
-
-    infer_types: optional bool
-      If true, will attemp to convert types from Python to QuasarDB natives types if
-      the provided dataframe has incompatible types. For example, a dataframe with integers
-      will automatically convert these to doubles if the QuasarDB table expects it.
-
-      **Important**: as conversions are expensive and often the majority of time spent
-      when inserting data into QuasarDB, we strongly recommend setting this to ``False``
-      for performance-sensitive code.
-
-    truncate: optional bool
-      Truncate (also referred to as upsert) the data in-place. Will detect time range to truncate
-      from the time range inside the dataframe.
-
-      Defaults to False.
-
-    _async: optional bool
-      If true, uses asynchronous insertion API where commits are buffered server-side and
-      acknowledged before they are written to disk. If you insert to the same table from
-      multiple processes, setting this to True may improve performance.
-
-      Defaults to False.
-
-    fast: optional bool
-      Whether to use 'fast push'. If you incrementally add small batches of data to table,
-      you may see better performance if you set this to True.
-
-      Defaults to False.
-
+      The shard size of the timeseries you wish to create when `create` is True.
     """
 
     # If dfs is a dict, we convert it to a list of tuples.
     if isinstance(dfs, dict):
         dfs = dfs.items()
 
+    if shard_size is not None and create == False:
+        raise ValueError("Invalid argument: shard size provided while create is False")
+
     # If the tables are provided as strings, we look them up.
     dfs_ = []
     for (table, df) in dfs:
@@ -378,11 +343,10 @@ def write_dataframes(
 
         dfs_.append((table, df))
 
-
     data_by_table = []
 
     for table, df in dfs_:
-        logger.info("quasardb.pandas.write_dataframe, create = %s, dtype = %s", create, dtype)
+        logger.debug("quasardb.pandas.write_dataframe, create = %s", create)
         assert isinstance(df, pd.DataFrame)
 
         # Create table if requested
@@ -409,14 +373,7 @@ def write_dataframes(
     return qdbnp.write_arrays(data_by_table, cluster,
                               table=None,
                               index=None,
-                              dtype=dtype,
-                              _async=_async,
-                              fast=fast,
-                              truncate=truncate,
-                              deduplicate=deduplicate,
-                              deduplication_mode=deduplication_mode,
-                              infer_types=infer_types,
-                              writer=writer)
+                              **kwargs)
 
 def write_dataframe(
         df,

quasardb/properties.cpp

@@ -0,0 +1,41 @@
+#include "properties.hpp"
+#include <qdb/properties.h>
+#include "detail/qdb_resource.hpp"
+
+namespace qdb
+{
+
+std::optional<std::string> properties::get(std::string const & key)
+{
+    detail::qdb_resource<char const> ret{*handle_};
+
+    auto err = qdb_user_properties_get(*handle_, key.c_str(), &ret);
+
+    if (err == qdb_e_alias_not_found) [[unlikely]]
+    {
+        return {};
+    }
+
+    qdb::qdb_throw_if_error(*handle_, err);
+
+    assert(ret != nullptr);
+
+    return std::string{ret};
+}
+
+void properties::put(std::string const & key, std::string const & value)
+{
+    qdb::qdb_throw_if_error(*handle_, qdb_user_properties_put(*handle_, key.c_str(), value.c_str()));
+}
+
+void properties::remove(std::string const & key)
+{
+    qdb::qdb_throw_if_error(*handle_, qdb_user_properties_remove(*handle_, key.c_str()));
+}
+
+void properties::clear()
+{
+    qdb::qdb_throw_if_error(*handle_, qdb_user_properties_remove_all(*handle_));
+}
+
+}; // namespace qdb