quasardb 3.13.6__tar.gz → 3.13.7__tar.gz

{quasardb-3.13.6 → quasardb-3.13.7}/LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2009-2022, quasardb SAS. All rights reserved.
+Copyright (c) 2009-2023, quasardb SAS. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 

{quasardb-3.13.6 → quasardb-3.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quasardb
-Version: 3.13.6
+Version: 3.13.7
 Summary: Python API for quasardb
 Home-page: https://www.quasardb.net/
 Author: quasardb SAS

{quasardb-3.13.6 → quasardb-3.13.7}/quasardb/__init__.py

@@ -1,6 +1,6 @@
 # pylint: disable=C0103,C0111,C0302,R0903
 
-# Copyright (c) 2009-2022, quasardb SAS. All rights reserved.
+# Copyright (c) 2009-2023, quasardb SAS. All rights reserved.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

{quasardb-3.13.6 → quasardb-3.13.7}/quasardb/numpy/__init__.py

@@ -81,6 +81,19 @@ class IncompatibleDtypeErrors(TypeError):
     def msg(self):
         return "\n".join(x.msg() for x in self.xs)
 
+class InvalidDataCardinalityError(ValueError):
+    """
+    Raised when the provided data arrays doesn't match the table's columns.
+    """
+    def __init__(self, data, cinfos):
+        self.data = data
+        self.cinfos = cinfos
+        super().__init__(self.msg())
+
+    def msg(self):
+        return "Provided data array length '{}' exceeds amount of table columns '{}', unable to map data to columns".format(len(self.data), len(self.cinfos))
+
+
 # Based on QuasarDB column types, which dtype do we accept?
 # First entry will always be the 'preferred' dtype, other ones
 # those that we can natively convert in native code.
@@ -259,6 +272,42 @@ def _coerce_deduplicate(deduplicate, deduplication_mode, columns):
 
     return deduplicate
 
+def _clean_nulls(xs, dtype):
+    """
+    Numpy's masked arrays have a downside that in case they're not able to convert a (masked!) value to
+    the desired dtype, they raise an error. So, for example, if I have a masked array of objects that
+    look like this
+
+    xs: [1.234 <pd.NA> 5.678]
+    mask: [1 0 1]
+
+    even though pd.NA is not "visible", because it cannot be converted to a float(), the operation will
+    fail!
+
+    This function fixes this by replacing the null values with an acceptable value that can always be
+    converted to the desired dtype.
+    """
+
+    assert ma.isMA(xs)
+
+    if xs.dtype is not np.dtype('object'):
+        return xs
+
+    fill_value = None
+    if dtype == np.float64 or dtype == np.float32 or dtype == np.float16:
+        fill_value = float('nan')
+    elif dtype == np.int64 or dtype == np.int32 or dtype == np.int16:
+        fill_value = -1
+    elif dtype == np.dtype('datetime64[ns]'):
+        fill_value = np.datetime64('nat')
+
+    mask = xs.mask
+    xs_ = xs.filled(fill_value)
+
+    return ma.array(xs_, mask=mask)
+
+
+
 def _coerce_data(data, dtype):
     """
     Coerces each numpy array of `data` to the dtype present in `dtype`.
@@ -271,25 +320,34 @@ def _coerce_data(data, dtype):
         data_ = data[i]
 
         if dtype_ is not None and dtypes_equal(data_.dtype, dtype_) == False:
+            data_ = _clean_nulls(data_, dtype_)
+
+            assert ma.isMA(data_)
+
             logger.debug("data for column with offset %d was provided in dtype '%s', but need '%s': converting data...", i, data_.dtype, dtype_)
 
-            logger.debug("type of data[%d] after: %s", i, type(data[i]))
-            logger.debug("size of data[%d] after: %s", i, ma.size(data[i]))
-            logger.debug("data of data[%d] after: %s", i, data[i])
+            logger.debug("dtype of data[%d] before: %s", i, data_.dtype)
+            logger.debug("type of data[%d] after: %s", i, type(data_))
+            logger.debug("size of data[%d] after: %s", i, ma.size(data_))
+            logger.debug("data of data[%d] after: %s", i, data_)
 
             try:
                 data[i] = data_.astype(dtype_)
             except TypeError as err:
                 # One 'bug' is that, if everything is masked, the underlying data type can be
                 # pretty much anything.
-                if not _is_all_masked(data_):
+                if _is_all_masked(data_):
+                    logger.debug("array completely empty, re-initializing to empty array of '%s'", dtype_)
+                    data[i] = ma.masked_all(ma.size(data_),
+                                            dtype=dtype_)
+
+                # Another 'bug' is that when the input data is objects, we may have null-like values (like pd.NA)
+                # that cannot easily be converted to, say, float.
+                else:
                     logger.error("An error occured while coercing input data type from dtype '%s' to dtype '%s': ", data_.dtype, dtype_)
                     logger.exception(err)
                     raise err
 
-                logger.debug("array completely empty, re-initializing to empty array of '%s'", dtype_)
-                data[i] = ma.masked_all(ma.size(data_),
-                                        dtype=dtype_)
             assert data[i].dtype.kind == dtype_.kind
 
             logger.debug("type of data[%d] after: %s", i, type(data[i]))
@@ -321,6 +379,13 @@ def _ensure_list(xs, cinfos):
     if isinstance(xs, list):
         return xs
 
+    if isinstance(xs, np.ndarray):
+        ret = []
+        for x in xs:
+            ret.append(x)
+
+        return ret
+
     # As we only accept list-likes or dicts as input data, it *must* be a dict at this
     # point
     assert isinstance(xs, dict)
@@ -612,6 +677,10 @@ def write_arrays(
         dtype = _add_desired_dtypes(dtype, cinfos)
 
     data = _ensure_list(data, cinfos)
+
+    if len(data) != len(cinfos):
+        raise InvalidDataCardinalityError(data, cinfos)
+
     data = ensure_ma(data, dtype=dtype)
     data = _coerce_data(data, dtype)
 

{quasardb-3.13.6 → quasardb-3.13.7}/quasardb/pandas/__init__.py

@@ -1,6 +1,6 @@
 # pylint: disable=C0103,C0111,C0302,R0903
 
-# Copyright (c) 2009-2022, quasardb SAS. All rights reserved.
+# Copyright (c) 2009-2023, quasardb SAS. All rights reserved.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

{quasardb-3.13.6 → quasardb-3.13.7}/quasardb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quasardb
-Version: 3.13.6
+Version: 3.13.7
 Summary: Python API for quasardb
 Home-page: https://www.quasardb.net/
 Author: quasardb SAS

quasardb-3.13.7/quasardb.egg-info/SOURCES.txt

@@ -0,0 +1,62 @@
+LICENSE.md
+README.md
+setup.cfg
+setup.py
+quasardb/__init__.py
+quasardb/firehose.py
+quasardb/pool.py
+quasardb/stats.py
+quasardb.egg-info/PKG-INFO
+quasardb.egg-info/SOURCES.txt
+quasardb.egg-info/dependency_links.txt
+quasardb.egg-info/not-zip-safe
+quasardb.egg-info/requires.txt
+quasardb.egg-info/top_level.txt
+quasardb/extensions/__init__.py
+quasardb/extensions/pinned_writer.py
+quasardb/numpy/__init__.py
+quasardb/pandas/__init__.py
+tests/test_basic.py
+tests/test_batch_inserter.py
+tests/test_bench_pinned_writer.py
+tests/test_blob.py
+tests/test_buf_size.py
+tests/test_connection.py
+tests/test_continuous.py
+tests/test_convert.py
+tests/test_double.py
+tests/test_entry.py
+tests/test_error.py
+tests/test_examples.py
+tests/test_expiry.py
+tests/test_firehose.py
+tests/test_getTimeout.py
+tests/test_info.py
+tests/test_integer.py
+tests/test_logging.py
+tests/test_node.py
+tests/test_numpy.py
+tests/test_option_client_max_parallelism.py
+tests/test_pandas.py
+tests/test_pandas_benchmark.py
+tests/test_perf.py
+tests/test_pinned_writer.py
+tests/test_pool.py
+tests/test_prefix.py
+tests/test_query.py
+tests/test_query_find.py
+tests/test_setCompression.py
+tests/test_setMaxCardinality.py
+tests/test_setTimeout.py
+tests/test_stats.py
+tests/test_string.py
+tests/test_suffix.py
+tests/test_table.py
+tests/test_table_blob.py
+tests/test_table_double.py
+tests/test_table_int64.py
+tests/test_table_reader.py
+tests/test_table_string.py
+tests/test_table_timestamp.py
+tests/test_tag.py
+tests/test_timestamp.py

{quasardb-3.13.6 → quasardb-3.13.7}/setup.py

@@ -21,7 +21,7 @@ from setuptools.command.install import install
 from pkg_resources import get_build_platform
 from wheel.bdist_wheel import bdist_wheel as old_bdist_wheel
 
-qdb_version = "3.13.6"
+qdb_version = "3.13.7"
 
 # package_modules are our 'extra' files. Our cmake configuration copies our QDB_API_LIB
 # into our source directory, and by adding this to `package_modules` we tell setuptools to
@@ -73,6 +73,9 @@ class CMakeBuild(build_ext):
         return out.decode().strip()
 
     def build_extension(self, ext):
+        self.do_build_extension(ext)
+
+    def do_build_extension(self, ext):
         extdir = os.path.join(
             os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))), 'quasardb')
 
@@ -82,7 +85,7 @@ class CMakeBuild(build_ext):
         cmake_args = [
             '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
             '-DPYTHON_EXECUTABLE=' + sys.executable,
-            '-DQDB_PY_VERSION=' + qdb_version,
+            '-DQDB_PY_VERSION=' + qdb_version
         ]
 
         if platform.system() == "Darwin":
@@ -95,12 +98,14 @@ class CMakeBuild(build_ext):
                             'CMAKE_C_COMPILER': ['-D', 'CMAKE_C_COMPILER={}'],
                             'CMAKE_CXX_COMPILER': ['-D', 'CMAKE_CXX_COMPILER={}'],
                             'QDB_LINKER': ['-D', 'QDB_LINKER={}'],
+                            'QDB_TESTS_ENABLED': ['-D', 'QDB_TESTS_ENABLED={}'],
                             'CMAKE_BUILD_TYPE': ['-D', 'CMAKE_BUILD_TYPE={}'],
                             'CMAKE_OSX_DEPLOYMENT_TARGET': ['-D', 'CMAKE_OSX_DEPLOYMENT_TARGET={}'],
                             'CMAKE_OSX_SYSROOT': ['-D', 'CMAKE_OSX_SYSROOT={}'],
                             'CMAKE_VERBOSE_MAKEFILE': ['-D', 'CMAKE_VERBOSE_MAKEFILE={}'],
                             }
-        default_proxy_vals = {'CMAKE_BUILD_TYPE': 'Release'}
+        default_proxy_vals = {'CMAKE_BUILD_TYPE': 'Release',
+                              'QDB_TESTS_ENABLED': 'OFF'}
 
         for (env_var, cmake_args_) in proxied_env_vars.items():
             default_ = default_proxy_vals.get(env_var, 0)

quasardb-3.13.7/tests/test_basic.py

@@ -0,0 +1,19 @@
+# pylint: disable=C0103,C0111,C0302,W0212
+import datetime
+
+import pytest
+import quasardb
+
+
+def test_build():
+    build = quasardb.build()
+    assert len(build) > 0
+
+
+def test_version():
+    build = quasardb.version()
+    assert len(build) > 0
+
+
+def test_can_purge_all(qdbd_secure_connection):
+    qdbd_secure_connection.purge_all(datetime.timedelta(minutes=1))

quasardb-3.13.7/tests/test_batch_inserter.py

@@ -0,0 +1,343 @@
+# pylint: disable=C0103,C0111,C0302,W0212
+from builtins import range as xrange, int as long  # pylint: disable=W0622
+from functools import reduce  # pylint: disable=W0622
+import datetime
+import test_table as tslib
+from time import sleep
+
+import pytest
+import quasardb
+import numpy as np
+
+
+def _row_insertion_method(
+        inserter,
+        dates,
+        doubles,
+        blobs,
+        strings,
+        integers,
+        timestamps,
+        symbols):
+    for i in range(len(dates)):
+        inserter.start_row(dates[i])
+        inserter.set_double(0, doubles[i])
+        inserter.set_blob(1, blobs[i])
+        inserter.set_string(2, strings[i])
+        inserter.set_int64(3, integers[i])
+        inserter.set_timestamp(4, timestamps[i])
+        inserter.set_string(5, symbols[i]) # symbol columns use string representation
+
+
+def _regular_push(inserter):
+    inserter.push()
+
+
+def _async_push(inserter):
+    inserter.push_async()
+    # Wait for push_async to complete
+    # Ideally we could be able to get the proper flush interval
+    sleep(15)
+
+
+def _fast_push(inserter):
+    inserter.push_fast()
+
+
+def _make_inserter_info(table):
+    return [quasardb.BatchColumnInfo(table.get_name(), tslib._double_col_name(table), 1000),
+            quasardb.BatchColumnInfo(
+                table.get_name(), tslib._blob_col_name(table), 1000),
+            quasardb.BatchColumnInfo(
+                table.get_name(), tslib._string_col_name(table), 1000),
+            quasardb.BatchColumnInfo(
+                table.get_name(), tslib._int64_col_name(table), 1000),
+            quasardb.BatchColumnInfo(table.get_name(), tslib._ts_col_name(table), 1000),
+            quasardb.BatchColumnInfo(table.get_name(), tslib._symbol_col_name(table), 1000)]
+
+
+def test_non_existing_bulk_insert(qdbd_connection, entry_name):
+    with pytest.raises(quasardb.AliasNotFoundError):
+        qdbd_connection.inserter(
+            [quasardb.BatchColumnInfo(entry_name, "col", 10)])
+
+
+def _generate_data(count, start=np.datetime64('2017-01-01', 'ns')):
+    doubles = np.random.uniform(-100.0, 100.0, count)
+    integers = np.random.randint(-100, 100, count)
+    blobs = np.array(list(np.random.bytes(np.random.randint(8, 16))
+                          for i in range(count)), 'O')
+    strings = np.array([("content_" + str(item)) for item in range(count)])
+    timestamps = tslib._generate_dates(
+        start + np.timedelta64('1', 'D'), count)
+    symbols = np.array([("symbol_" + str(item)) for item in range(count)])
+
+    return (doubles, integers, blobs, strings, timestamps, symbols)
+
+
+def _set_batch_inserter_data(inserter, intervals, data, start=0):
+    (doubles, integers, blobs, strings, timestamps, symbols) = data
+
+    for i in range(start, len(intervals)):
+        inserter.start_row(intervals[i])
+        inserter.set_double(0, doubles[i])
+        inserter.set_blob(1, blobs[i])
+        inserter.set_string(2, strings[i])
+        inserter.set_int64(3, integers[i])
+        inserter.set_timestamp(4, timestamps[i])
+        inserter.set_string(5, symbols[i])
+
+
+def _assert_results(table, intervals, data):
+    (doubles, integers, blobs, strings, timestamps, symbols) = data
+
+    whole_range = (intervals[0], intervals[-1:][0] + np.timedelta64(2, 's'))
+    results = table.double_get_ranges(
+        tslib._double_col_name(table), [whole_range])
+
+    np.testing.assert_array_equal(results[0], intervals)
+    np.testing.assert_array_equal(results[1], doubles)
+
+    results = table.blob_get_ranges(tslib._blob_col_name(table), [whole_range])
+    np.testing.assert_array_equal(results[0], intervals)
+    np.testing.assert_array_equal(results[1], blobs)
+
+    results = table.string_get_ranges(
+        tslib._string_col_name(table), [whole_range])
+    np.testing.assert_array_equal(results[0], intervals)
+    np.testing.assert_array_equal(results[1], strings)
+
+    results = table.int64_get_ranges(
+        tslib._int64_col_name(table), [whole_range])
+    np.testing.assert_array_equal(results[0], intervals)
+    np.testing.assert_array_equal(results[1], integers)
+
+    results = table.timestamp_get_ranges(
+        tslib._ts_col_name(table), [whole_range])
+    np.testing.assert_array_equal(results[0], intervals)
+    np.testing.assert_array_equal(results[1], timestamps)
+
+    results = table.string_get_ranges(
+        tslib._symbol_col_name(table), [whole_range])
+    np.testing.assert_array_equal(results[0], intervals)
+    np.testing.assert_array_equal(results[1], symbols)
+
+
+def _test_with_table(
+        inserter,
+        table,
+        intervals,
+        push_method=_regular_push,
+        data=None):
+
+    if data is None:
+        data = _generate_data(len(intervals))
+
+    # range is right exclusive, so the timestamp has to be beyond
+    whole_range = (intervals[0], intervals[-1:][0] + np.timedelta64(2, 's'))
+
+    (doubles, integers, blobs, strings, timestamps, symbols) = data
+
+    _set_batch_inserter_data(inserter, intervals, data)
+
+    # before the push, there is nothing
+    results = table.double_get_ranges(
+        tslib._double_col_name(table), [whole_range])
+    assert len(results[0]) == 0
+
+    results = table.blob_get_ranges(tslib._blob_col_name(table), [whole_range])
+    assert len(results[0]) == 0
+
+    results = table.string_get_ranges(
+        tslib._string_col_name(table), [whole_range])
+    assert len(results[0]) == 0
+
+    results = table.int64_get_ranges(
+        tslib._int64_col_name(table), [whole_range])
+    assert len(results[0]) == 0
+
+    results = table.timestamp_get_ranges(
+        tslib._ts_col_name(table), [whole_range])
+    assert len(results[0]) == 0
+
+    results = table.string_get_ranges(
+        tslib._symbol_col_name(table), [whole_range])
+    assert len(results[0]) == 0
+
+    # after push, there is everything
+    push_method(inserter)
+    if push_method == _async_push:
+        sleep(20)
+
+    _assert_results(table, intervals, data)
+
+    return doubles, blobs, strings, integers, timestamps, symbols
+
+
+def test_successful_bulk_row_insert(qdbd_connection, table, many_intervals):
+    inserter = qdbd_connection.inserter(_make_inserter_info(table))
+
+    _test_with_table(
+        inserter,
+        table,
+        many_intervals,
+        _regular_push)
+
+
+def test_successful_secure_bulk_row_insert(
+        qdbd_secure_connection,
+        secure_table,
+        many_intervals):
+    inserter = qdbd_secure_connection.inserter(
+        _make_inserter_info(secure_table))
+
+    _test_with_table(
+        inserter,
+        secure_table,
+        many_intervals,
+        _regular_push)
+
+
+
+@pytest.mark.skip(reason="Skip slow tests")
+def test_successful_async_bulk_row_insert(
+        qdbd_connection, table, many_intervals):
+
+    # Same test as `test_successful_bulk_row_insert` but using `push_async` to push the entries
+    # This allows us to test the `push_async` feature
+
+    inserter = qdbd_connection.inserter(_make_inserter_info(table))
+    _test_with_table(
+        inserter,
+        table,
+        many_intervals,
+        _async_push)
+
+
+def test_successful_fast_bulk_row_insert(
+        qdbd_connection, table, many_intervals):
+    # Same test as `test_successful_bulk_row_insert` but using `push_async` to push the entries
+    # This allows us to test the `push_async` feature
+
+    inserter = qdbd_connection.inserter(_make_inserter_info(table))
+    _test_with_table(
+        inserter,
+        table,
+        many_intervals,
+        _fast_push)
+
+
+def test_failed_local_table_with_wrong_columns(qdbd_connection, entry_name):
+    columns = [quasardb.BatchColumnInfo(entry_name, "1000flavorsofwrong", 10)]
+    with pytest.raises(quasardb.AliasNotFoundError):
+        qdbd_connection.inserter(columns)
+
+
+def test_push_truncate_implicit_range(qdbd_connection, table, many_intervals):
+
+    whole_range = (
+        many_intervals[0], many_intervals[-1:][0] + np.timedelta64(2, 's'))
+
+    # Generate our dataset
+    data = _generate_data(len(many_intervals))
+    # (doubles, integers, blobs, strings, timestamps) = data
+    (doubles, _, _, _, _, _) = data
+
+    # Insert once
+    inserter = qdbd_connection.inserter(_make_inserter_info(table))
+    _set_batch_inserter_data(inserter, many_intervals, data)
+    inserter.push()
+
+    # Compare results, should be equal
+    results = table.double_get_ranges(
+        tslib._double_col_name(table), [whole_range])
+
+    np.testing.assert_array_equal(results[0], many_intervals)
+    np.testing.assert_array_equal(results[1], doubles)
+
+    # Insert regular, twice
+    _set_batch_inserter_data(inserter, many_intervals, data)
+    inserter.push()
+
+    # Compare results, should now have the same data twice
+    results = table.double_get_ranges(
+        tslib._double_col_name(table), [whole_range])
+
+    assert len(results[1]) == 2 * len(doubles)
+
+    # Insert truncate, should now have original data again
+    _set_batch_inserter_data(inserter, many_intervals, data)
+    inserter.push_truncate()
+
+    # Verify results, truncating should now make things the same
+    # as the beginning again.
+    results = table.double_get_ranges(
+        tslib._double_col_name(table), [whole_range])
+
+    np.testing.assert_array_equal(results[0], many_intervals)
+    np.testing.assert_array_equal(results[1], doubles)
+
+
+def test_push_truncate_explicit_range(qdbd_connection, table, many_intervals):
+
+    whole_range = (
+        many_intervals[0], many_intervals[-1:][0] + np.timedelta64(2, 's'))
+
+    # Generate our dataset
+    data = _generate_data(len(many_intervals))
+    # (doubles, integers, blobs, strings, timestamps) = data
+    (doubles, _, _, _, _, _) = data
+
+    inserter = qdbd_connection.inserter(_make_inserter_info(table))
+
+    # Insert once
+    truncate_range = (whole_range[0],
+                      whole_range[1] + np.timedelta64(1, 'ns'))
+
+    _set_batch_inserter_data(inserter, many_intervals, data)
+    inserter.push()
+
+    # Verify results, truncating should now make things the same
+    # as the beginning again.
+    results = table.double_get_ranges(
+        tslib._double_col_name(table), [whole_range])
+
+    np.testing.assert_array_equal(results[0], many_intervals)
+    np.testing.assert_array_equal(results[1], doubles)
+
+    # If we now set the same data, skip the first element, but keep
+    # the same time range, the first element will *not* be present in
+    # the resulting dataset.
+    _set_batch_inserter_data(inserter, many_intervals, data, start=1)
+    inserter.push_truncate(range=truncate_range)
+
+    # Verify results, truncating should now make things the same
+    # as the beginning again.
+    results = table.double_get_ranges(
+        tslib._double_col_name(table), [whole_range])
+
+    np.testing.assert_array_equal(results[0], many_intervals[1:])
+    np.testing.assert_array_equal(results[1], doubles[1:])
+
+
+def test_push_truncate_throws_error_on_invalid_range(
+        qdbd_connection, table, many_intervals):
+    print("table = {}".format(table.get_name()))
+    whole_range = (
+        many_intervals[0], many_intervals[-1:][0] + np.timedelta64(2, 's'))
+
+    # Generate our dataset
+    data = _generate_data(len(many_intervals))
+    # (doubles, integers, blobs, strings, timestamps) = data
+    (_, _, _, _, _, _) = data
+
+    # Insert truncate with explicit timerange, we point the start right after the
+    # first element in our dataset. This means that the range does not overlap all
+    # the data anymore.
+    truncate_range = (whole_range[0] + np.timedelta64(1, 'ns'),
+                      whole_range[1] + np.timedelta64(1, 'ns'))
+
+    inserter = qdbd_connection.inserter(_make_inserter_info(table))
+    _set_batch_inserter_data(inserter, many_intervals, data)
+    with pytest.raises(quasardb.InvalidArgumentError):
+        inserter.push_truncate(range=truncate_range)