quasardb 3.13.5.post3__tar.gz → 3.13.7__tar.gz

{quasardb-3.13.5.post3 → quasardb-3.13.7}/LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2009-2022, quasardb SAS. All rights reserved.
+Copyright (c) 2009-2023, quasardb SAS. All rights reserved.
 
 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 

{quasardb-3.13.5.post3 → quasardb-3.13.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quasardb
-Version: 3.13.5.post3
+Version: 3.13.7
 Summary: Python API for quasardb
 Home-page: https://www.quasardb.net/
 Author: quasardb SAS
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Database
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: License :: OSI Approved :: BSD License

{quasardb-3.13.5.post3 → quasardb-3.13.7}/quasardb/__init__.py

@@ -1,6 +1,6 @@
 # pylint: disable=C0103,C0111,C0302,R0903
 
-# Copyright (c) 2009-2022, quasardb SAS. All rights reserved.
+# Copyright (c) 2009-2023, quasardb SAS. All rights reserved.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without

{quasardb-3.13.5.post3 → quasardb-3.13.7}/quasardb/numpy/__init__.py

@@ -81,6 +81,19 @@ class IncompatibleDtypeErrors(TypeError):
     def msg(self):
         return "\n".join(x.msg() for x in self.xs)
 
+class InvalidDataCardinalityError(ValueError):
+    """
+    Raised when the provided data arrays doesn't match the table's columns.
+    """
+    def __init__(self, data, cinfos):
+        self.data = data
+        self.cinfos = cinfos
+        super().__init__(self.msg())
+
+    def msg(self):
+        return "Provided data array length '{}' exceeds amount of table columns '{}', unable to map data to columns".format(len(self.data), len(self.cinfos))
+
+
 # Based on QuasarDB column types, which dtype do we accept?
 # First entry will always be the 'preferred' dtype, other ones
 # those that we can natively convert in native code.
@@ -233,25 +246,67 @@ def _validate_dtypes(data, columns):
     if len(errors) > 0:
         raise IncompatibleDtypeErrors(errors)
 
-
-def _validate_drop_duplicates(drop_duplicates, columns):
+def _coerce_deduplicate(deduplicate, deduplication_mode, columns):
     """
-    Throws an error when 'drop_duplicates' refers to a column not present in the table.
+    Throws an error when 'deduplicate' options are incorrect.
     """
-    if isinstance(drop_duplicates, bool):
-        return True
+    cnames = [cname for (cname, ctype) in columns]
 
-    if not isinstance(drop_duplicates, list):
-        raise TypeError("drop_duplicates should be either a bool or a list, got: " + type(drop_duplicates))
+    if deduplication_mode not in ['drop', 'upsert']:
+        raise RuntimeError("deduplication_mode should be one of ['drop', 'upsert'], got: {}".format(deduplication_mode))
 
-    cnames = [cname for (cname, ctype) in columns]
+    if isinstance(deduplicate, bool):
+        return deduplicate
+
+    # Special value of $timestamp, hardcoded
+    if isinstance(deduplicate, str) and deduplicate == '$timestamp':
+        deduplicate = ['$timestamp']
+    cnames.append('$timestamp')
+
+    if not isinstance(deduplicate, list):
+        raise TypeError("drop_duplicates should be either a bool or a list, got: " + type(deduplicate))
 
-    for column_name in drop_duplicates:
+    for column_name in deduplicate:
         if not column_name in cnames:
             raise RuntimeError("Provided deduplication column name '{}' not found in table columns.".format(column_name))
 
+    return deduplicate
+
+def _clean_nulls(xs, dtype):
+    """
+    Numpy's masked arrays have a downside that in case they're not able to convert a (masked!) value to
+    the desired dtype, they raise an error. So, for example, if I have a masked array of objects that
+    look like this
+
+    xs: [1.234 <pd.NA> 5.678]
+    mask: [1 0 1]
+
+    even though pd.NA is not "visible", because it cannot be converted to a float(), the operation will
+    fail!
+
+    This function fixes this by replacing the null values with an acceptable value that can always be
+    converted to the desired dtype.
+    """
+
+    assert ma.isMA(xs)
+
+    if xs.dtype is not np.dtype('object'):
+        return xs
+
+    fill_value = None
+    if dtype == np.float64 or dtype == np.float32 or dtype == np.float16:
+        fill_value = float('nan')
+    elif dtype == np.int64 or dtype == np.int32 or dtype == np.int16:
+        fill_value = -1
+    elif dtype == np.dtype('datetime64[ns]'):
+        fill_value = np.datetime64('nat')
+
+    mask = xs.mask
+    xs_ = xs.filled(fill_value)
+
+    return ma.array(xs_, mask=mask)
+
 
-    return True
 
 def _coerce_data(data, dtype):
     """
@@ -265,25 +320,34 @@ def _coerce_data(data, dtype):
         data_ = data[i]
 
         if dtype_ is not None and dtypes_equal(data_.dtype, dtype_) == False:
+            data_ = _clean_nulls(data_, dtype_)
+
+            assert ma.isMA(data_)
+
             logger.debug("data for column with offset %d was provided in dtype '%s', but need '%s': converting data...", i, data_.dtype, dtype_)
 
-            logger.debug("type of data[%d] after: %s", i, type(data[i]))
-            logger.debug("size of data[%d] after: %s", i, ma.size(data[i]))
-            logger.debug("data of data[%d] after: %s", i, data[i])
+            logger.debug("dtype of data[%d] before: %s", i, data_.dtype)
+            logger.debug("type of data[%d] after: %s", i, type(data_))
+            logger.debug("size of data[%d] after: %s", i, ma.size(data_))
+            logger.debug("data of data[%d] after: %s", i, data_)
 
             try:
                 data[i] = data_.astype(dtype_)
             except TypeError as err:
                 # One 'bug' is that, if everything is masked, the underlying data type can be
                 # pretty much anything.
-                if not _is_all_masked(data_):
+                if _is_all_masked(data_):
+                    logger.debug("array completely empty, re-initializing to empty array of '%s'", dtype_)
+                    data[i] = ma.masked_all(ma.size(data_),
+                                            dtype=dtype_)
+
+                # Another 'bug' is that when the input data is objects, we may have null-like values (like pd.NA)
+                # that cannot easily be converted to, say, float.
+                else:
                     logger.error("An error occured while coercing input data type from dtype '%s' to dtype '%s': ", data_.dtype, dtype_)
                     logger.exception(err)
                     raise err
 
-                logger.debug("array completely empty, re-initializing to empty array of '%s'", dtype_)
-                data[i] = ma.masked_all(ma.size(data_),
-                                        dtype=dtype_)
             assert data[i].dtype.kind == dtype_.kind
 
             logger.debug("type of data[%d] after: %s", i, type(data[i]))
@@ -315,6 +379,13 @@ def _ensure_list(xs, cinfos):
     if isinstance(xs, list):
         return xs
 
+    if isinstance(xs, np.ndarray):
+        ret = []
+        for x in xs:
+            ret.append(x)
+
+        return ret
+
     # As we only accept list-likes or dicts as input data, it *must* be a dict at this
     # point
     assert isinstance(xs, dict)
@@ -494,7 +565,8 @@ def write_arrays(
         _async = False,
         fast = False,
         truncate = False,
-        drop_duplicates=False,
+        deduplicate=False,
+        deduplication_mode='drop',
         infer_types = True,
         writer = None):
     """
@@ -537,13 +609,20 @@ def write_arrays(
       If a dtype for a column is provided in this argument, and infer_types is also
       True, this argument takes precedence.
 
-    drop_duplicates: bool or list[str]
+    deduplicate: bool or list[str]
       Enables server-side deduplication of data when it is written into the table.
       When True, automatically deduplicates rows when all values of a row are identical.
       When a list of strings is provided, deduplicates only based on the values of
       these columns.
 
-    Defaults to False.
+      Defaults to False.
+
+    deduplication_mode: 'drop' or 'upsert'
+      When `deduplicate` is enabled, decides how deduplication is performed. 'drop' means
+      any newly written duplicates are dropped, where 'upsert' means that the previously
+      written data is updated to reflect the new data.
+
+      Defaults to 'drop'.
 
     infer_types: optional bool
       If true, will attemp to convert types from Python to QuasarDB natives types if
@@ -598,6 +677,10 @@ def write_arrays(
         dtype = _add_desired_dtypes(dtype, cinfos)
 
     data = _ensure_list(data, cinfos)
+
+    if len(data) != len(cinfos):
+        raise InvalidDataCardinalityError(data, cinfos)
+
     data = ensure_ma(data, dtype=dtype)
     data = _coerce_data(data, dtype)
 
@@ -607,7 +690,7 @@ def write_arrays(
     # code as it generally makes for somewhat better error context.
     _validate_dtypes(data, cinfos)
 
-    _validate_drop_duplicates(drop_duplicates, cinfos)
+    deduplicate = _coerce_deduplicate(deduplicate, deduplication_mode, cinfos)
 
     write_with = {
         quasardb.ColumnType.Double: writer.set_double_column,
@@ -633,15 +716,15 @@ def write_arrays(
     start = time.time()
 
     if fast is True:
-        writer.push_fast(drop_duplicates=drop_duplicates)
+        writer.push_fast(deduplicate=deduplicate, deduplication_mode=deduplication_mode)
     elif truncate is True:
-        writer.push_truncate(drop_duplicates=drop_duplicates)
+        writer.push_truncate(deduplicate=deduplicate, deduplication_mode=deduplication_mode)
     elif isinstance(truncate, tuple):
-        writer.push_truncate(range=truncate, drop_duplicates=drop_duplicates)
+        writer.push_truncate(range=truncate, deduplicate=deduplicate, deduplication_mode=deduplication_mode)
     elif _async is True:
-        writer.push_async(drop_duplicates=drop_duplicates)
+        writer.push_async(deduplicate=deduplicate, deduplication_mode=deduplication_mode)
     else:
-        writer.push(drop_duplicates=drop_duplicates)
+        writer.push(deduplicate=deduplicate, deduplication_mode=deduplication_mode)
 
     logger.debug("pushed %d rows in %s seconds",
                  len(index), (time.time() - start))

{quasardb-3.13.5.post3 → quasardb-3.13.7}/quasardb/pandas/__init__.py

@@ -1,6 +1,6 @@
 # pylint: disable=C0103,C0111,C0302,R0903
 
-# Copyright (c) 2009-2022, quasardb SAS. All rights reserved.
+# Copyright (c) 2009-2023, quasardb SAS. All rights reserved.
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -293,7 +293,8 @@ def write_dataframe(
         _async=False,
         fast=False,
         truncate=False,
-        drop_duplicates=False,
+        deduplicate=False,
+        deduplication_mode='drop',
         infer_types=True,
         writer=None):
     """
@@ -326,7 +327,7 @@ def write_dataframe(
     shard_size: optional datetime.timedelta
       The shard size of the timeseries you wish to create.
 
-    drop_duplicates: bool or list[str]
+    deduplicate: bool or list[str]
       Enables server-side deduplication of data when it is written into the table.
       When True, automatically deduplicates rows when all values of a row are identical.
       When a list of strings is provided, deduplicates only based on the values of
@@ -399,7 +400,8 @@ def write_dataframe(
                               _async=_async,
                               fast=fast,
                               truncate=truncate,
-                              drop_duplicates=drop_duplicates,
+                              deduplicate=deduplicate,
+                              deduplication_mode=deduplication_mode,
                               infer_types=infer_types,
                               writer=writer)
 

{quasardb-3.13.5.post3 → quasardb-3.13.7}/quasardb/stats.py

@@ -63,39 +63,91 @@ def of_node(dconn):
 
     return ret
 
+_stat_types = {'node_id': ('constant', None),
+               'operating_system': ('constant', None),
+               'partitions_count': ('constant', 'count'),
 
-_stat_types = {'requests.bytes_out': 'counter',
-               'requests.bytes_in': 'counter',
-               'async_pipelines.merge.bucket_count': 'counter',
-               'async_pipelines.merge.duration_us': 'counter',
-               'async_pipelines.write.successes_count': 'counter',
-               'async_pipelines.write.failures_count': 'counter',
-               'async_pipelines.write.time_us': 'counter'}
+               'cpu.system': ('counter', 'ns'),
+               'cpu.user': ('counter', 'ns'),
+               'cpu.idle': ('counter', 'ns'),
+               'startup': ('constant', None),
+               'startup_time': ('constant', None),
+               'shutdown_time': ('constant', None),
 
+               'network.current_users_count': ('gauge', 'count'),
+               'hardware_concurrency': ('gauge', 'count'),
+
+               'check.online': ('gauge', 'count'),
+               'check.duration_ms': ('constant', 'ms'),
+
+               'requests.bytes_in': ('counter', 'bytes'),
+               'requests.bytes_out': ('counter', 'bytes'),
+               'requests.errors_count': ('counter', 'count'),
+               'requests.successes_count': ('counter', 'count'),
+               'requests.total_count': ('counter', 'count'),
+
+               'async_pipelines.merge.bucket_count': ('counter', 'count'),
+               'async_pipelines.merge.duration_us': ('counter', 'us'),
+               'async_pipelines.write.successes_count': ('counter', 'count'),
+               'async_pipelines.write.failures_count': ('counter', 'count'),
+               'async_pipelines.write.time_us': ('counter', 'us'),
+
+               'async_pipelines.merge.max_bucket_count': ('gauge', 'count'),
+               'async_pipelines.merge.max_depth_count': ('gauge', 'count'),
+               'async_pipelines.merge.requests_count': ('counter', 'count'),
+
+               'evicted.count': ('counter', 'count'),
+               'pageins.count': ('counter', 'count'),
+
+               }
 
 async_pipeline_bytes_pattern  = re.compile(r'async_pipelines.pipe_[0-9]+.merge_map.bytes')
 async_pipeline_count_pattern  = re.compile(r'async_pipelines.pipe_[0-9]+.merge_map.count')
 
-
 def _stat_type(stat_id):
+    if stat_id in _stat_types:
+        return _stat_types[stat_id]
+    elif stat_id.endswith('total_ns'):
+        return ('counter', 'ns')
+    elif stat_id.endswith('total_bytes'):
+        return ('counter', 'bytes')
+    elif stat_id.endswith('read_bytes'):
+        return ('counter', 'bytes')
+    elif stat_id.endswith('written_bytes'):
+        return ('counter', 'bytes')
+    elif stat_id.endswith('total_count'):
+        return ('counter', 'count')
+    elif stat_id.startswith('network.sessions.'):
+        return ('gauge', 'count')
+    elif stat_id.startswith('memory.'):
+        # memory statistics are all gauges i think, describes how much memory currently allocated where
+        return ('gauge', 'bytes')
+    elif stat_id.startswith('persistence.') or stat_id.startswith('disk'):
+        # persistence are also all gauges, describes mostly how much is currently available/used on storage
+        return ('gauge', 'bytes')
+    elif stat_id.startswith('license.'):
+        return ('gauge', None)
+    elif stat_id.startswith('engine_'):
+        return ('constant', None)
+    elif async_pipeline_bytes_pattern.match(stat_id):
+        return ('gauge', 'bytes')
+    elif async_pipeline_count_pattern.match(stat_id):
+        return ('gauge', 'count')
+    else:
+        return None
+
+def stat_type(stat_id):
     """
     Returns the statistic type by a stat id. Returns one of:
 
      - 'gauge'
      - 'counter'
+     - None in case of unrecognized statistics
 
     This is useful for determining which value should be reported in a dashboard.
     """
-    if stat_id in _stat_types:
-        return _stat_types[stat_id]
-    elif stat_id.endswith('total_ns'):
-        return 'counter'
-    elif async_pipeline_bytes_pattern.match(stat_id):
-        return 'gauge'
-    elif async_pipeline_count_pattern.match(stat_id):
-        return 'gauge'
-    else:
-        return None
+    return _stat_type(stat_id)
+
 
 def _calculate_delta_stat(stat_id, prev, cur):
     logger.info("calculating delta for stat_id = {}, prev = {}. cur = {}".format(stat_id, prev, cur))
@@ -137,14 +189,20 @@ def calculate_delta(prev, cur):
 
     return ret
 
+def _clean_blob(x):
+    x_ = x.decode('utf-8', 'replace')
+
+    # remove trailing zero-terminator
+    return ''.join(c for c in x_ if ord(c) != 0)
+
+
 def _get_stat(dconn, k):
     # Ugly, but works: try to retrieve as integer, if not an int, retrieve as
     # blob
     try:
         return dconn.integer(k).get()
     except quasardb.quasardb.AliasNotFoundError:
-        blob = dconn.blob(k).get()
-        return blob.decode('utf-8', 'replace')
+        return _clean_blob(dconn.blob(k).get())
 
 def _by_uid(stats):
     xs = {}

{quasardb-3.13.5.post3 → quasardb-3.13.7}/quasardb.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: quasardb
-Version: 3.13.5.post3
+Version: 3.13.7
 Summary: Python API for quasardb
 Home-page: https://www.quasardb.net/
 Author: quasardb SAS
@@ -19,6 +19,7 @@ Classifier: Programming Language :: Python :: 3.7
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Classifier: Topic :: Database
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: License :: OSI Approved :: BSD License

quasardb-3.13.7/quasardb.egg-info/SOURCES.txt

@@ -0,0 +1,62 @@
+LICENSE.md
+README.md
+setup.cfg
+setup.py
+quasardb/__init__.py
+quasardb/firehose.py
+quasardb/pool.py
+quasardb/stats.py
+quasardb.egg-info/PKG-INFO
+quasardb.egg-info/SOURCES.txt
+quasardb.egg-info/dependency_links.txt
+quasardb.egg-info/not-zip-safe
+quasardb.egg-info/requires.txt
+quasardb.egg-info/top_level.txt
+quasardb/extensions/__init__.py
+quasardb/extensions/pinned_writer.py
+quasardb/numpy/__init__.py
+quasardb/pandas/__init__.py
+tests/test_basic.py
+tests/test_batch_inserter.py
+tests/test_bench_pinned_writer.py
+tests/test_blob.py
+tests/test_buf_size.py
+tests/test_connection.py
+tests/test_continuous.py
+tests/test_convert.py
+tests/test_double.py
+tests/test_entry.py
+tests/test_error.py
+tests/test_examples.py
+tests/test_expiry.py
+tests/test_firehose.py
+tests/test_getTimeout.py
+tests/test_info.py
+tests/test_integer.py
+tests/test_logging.py
+tests/test_node.py
+tests/test_numpy.py
+tests/test_option_client_max_parallelism.py
+tests/test_pandas.py
+tests/test_pandas_benchmark.py
+tests/test_perf.py
+tests/test_pinned_writer.py
+tests/test_pool.py
+tests/test_prefix.py
+tests/test_query.py
+tests/test_query_find.py
+tests/test_setCompression.py
+tests/test_setMaxCardinality.py
+tests/test_setTimeout.py
+tests/test_stats.py
+tests/test_string.py
+tests/test_suffix.py
+tests/test_table.py
+tests/test_table_blob.py
+tests/test_table_double.py
+tests/test_table_int64.py
+tests/test_table_reader.py
+tests/test_table_string.py
+tests/test_table_timestamp.py
+tests/test_tag.py
+tests/test_timestamp.py

{quasardb-3.13.5.post3 → quasardb-3.13.7}/setup.py

@@ -21,7 +21,7 @@ from setuptools.command.install import install
 from pkg_resources import get_build_platform
 from wheel.bdist_wheel import bdist_wheel as old_bdist_wheel
 
-qdb_version = "3.13.5.post3"
+qdb_version = "3.13.7"
 
 # package_modules are our 'extra' files. Our cmake configuration copies our QDB_API_LIB
 # into our source directory, and by adding this to `package_modules` we tell setuptools to
@@ -73,6 +73,9 @@ class CMakeBuild(build_ext):
         return out.decode().strip()
 
     def build_extension(self, ext):
+        self.do_build_extension(ext)
+
+    def do_build_extension(self, ext):
         extdir = os.path.join(
             os.path.abspath(os.path.dirname(self.get_ext_fullpath(ext.name))), 'quasardb')
 
@@ -82,7 +85,7 @@ class CMakeBuild(build_ext):
         cmake_args = [
             '-DCMAKE_LIBRARY_OUTPUT_DIRECTORY=' + extdir,
             '-DPYTHON_EXECUTABLE=' + sys.executable,
-            '-DQDB_PY_VERSION=' + qdb_version,
+            '-DQDB_PY_VERSION=' + qdb_version
         ]
 
         if platform.system() == "Darwin":
@@ -95,12 +98,14 @@ class CMakeBuild(build_ext):
                             'CMAKE_C_COMPILER': ['-D', 'CMAKE_C_COMPILER={}'],
                             'CMAKE_CXX_COMPILER': ['-D', 'CMAKE_CXX_COMPILER={}'],
                             'QDB_LINKER': ['-D', 'QDB_LINKER={}'],
+                            'QDB_TESTS_ENABLED': ['-D', 'QDB_TESTS_ENABLED={}'],
                             'CMAKE_BUILD_TYPE': ['-D', 'CMAKE_BUILD_TYPE={}'],
                             'CMAKE_OSX_DEPLOYMENT_TARGET': ['-D', 'CMAKE_OSX_DEPLOYMENT_TARGET={}'],
                             'CMAKE_OSX_SYSROOT': ['-D', 'CMAKE_OSX_SYSROOT={}'],
                             'CMAKE_VERBOSE_MAKEFILE': ['-D', 'CMAKE_VERBOSE_MAKEFILE={}'],
                             }
-        default_proxy_vals = {'CMAKE_BUILD_TYPE': 'Release'}
+        default_proxy_vals = {'CMAKE_BUILD_TYPE': 'Release',
+                              'QDB_TESTS_ENABLED': 'OFF'}
 
         for (env_var, cmake_args_) in proxied_env_vars.items():
             default_ = default_proxy_vals.get(env_var, 0)
@@ -216,6 +221,7 @@ setup(name=package_name,
           'Programming Language :: Python :: 3.8',
           'Programming Language :: Python :: 3.9',
           'Programming Language :: Python :: 3.10',
+          'Programming Language :: Python :: 3.11',
           'Topic :: Database',
           'Topic :: Software Development :: Libraries :: Python Modules',
           "License :: OSI Approved :: BSD License",

quasardb-3.13.7/tests/test_basic.py

@@ -0,0 +1,19 @@
+# pylint: disable=C0103,C0111,C0302,W0212
+import datetime
+
+import pytest
+import quasardb
+
+
+def test_build():
+    build = quasardb.build()
+    assert len(build) > 0
+
+
+def test_version():
+    build = quasardb.version()
+    assert len(build) > 0
+
+
+def test_can_purge_all(qdbd_secure_connection):
+    qdbd_secure_connection.purge_all(datetime.timedelta(minutes=1))