pyreindexer 0.5.100000__pp38-pypy38_pp73-macosx_15_0_arm64.whl

pyreindexer/CMakeLists.txt

@@ -0,0 +1,91 @@
+cmake_minimum_required(VERSION 3.18)
+
+project(pyreindexer)
+
+set(CMAKE_CXX_STANDARD 20)
+set(CMAKE_CXX_STANDARD_REQUIRED ON)
+if(NOT CMAKE_BUILD_TYPE)
+    set(CMAKE_BUILD_TYPE "RelWithDebInfo")
+endif()
+
+enable_testing()
+
+option(CIBUILDWHEEL "Is cibuildwheel environment" OFF)
+if(DEFINED ENV{CIBUILDWHEEL} AND "$ENV{CIBUILDWHEEL}" STREQUAL "1")
+    set(CIBUILDWHEEL ON)
+endif()
+
+set(PY_MIN_VERSION 3.8)
+if(CIBUILDWHEEL AND APPLE)
+    file(GLOB PYTHON_INCLUDE_DIRS "/Library/Frameworks/Python.framework/Versions/*/include/python*")
+    include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
+elseif(CIBUILDWHEEL)
+    file(GLOB PYTHON_INCLUDE_DIRS "/opt/python/*/include/python*")
+    include_directories(SYSTEM ${PYTHON_INCLUDE_DIRS})
+else()
+    find_package(Python3 ${PY_MIN_VERSION} REQUIRED COMPONENTS Development)
+endif()
+
+set(RX_MIN_VERSION 5.7.0)
+find_package(reindexer CONFIG ${RX_MIN_VERSION} REQUIRED)
+
+option(WITH_GCOV "Enable instrumented code coverage build" OFF)
+
+set(LIB_BUILTIN_NAME "rawpyreindexerb")
+set(LIB_CPROTO_NAME "rawpyreindexerc")
+set(LIBS_EXT ".so")
+
+set(LIBSRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/src)
+set(RESOURCES_DIR ${CMAKE_CURRENT_SOURCE_DIR}/lib/include)
+
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wall -Wextra -Werror -Wswitch-enum")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20 -Wall -Wextra -Werror -Wswitch-enum -Wno-unused-parameter -fexceptions")
+string(REPLACE "-O2" "-O3" CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO}")
+
+if(CIBUILDWHEEL AND NOT APPLE)
+    set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -s")
+    set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s")
+endif()
+
+file(GLOB_RECURSE SRCS ${RESOURCES_DIR}/*.cc ${LIBSRC_DIR}/*.cc)
+
+include_directories(
+        ${LIBSRC_DIR}
+        ${RESOURCES_DIR}
+        ${Python3_INCLUDE_DIRS}
+        ${REINDEXER_INCLUDE_DIRS})
+
+link_directories(
+    ${REINDEXER_LIB_DIR}
+    ${REINDEXER_LINK_DIRS})
+
+add_library(${LIB_BUILTIN_NAME} SHARED ${SRCS})
+add_library(${LIB_CPROTO_NAME} SHARED ${SRCS})
+
+target_link_libraries(${LIB_BUILTIN_NAME} ${Python3_LIBRARIES} ${REINDEXER_LIBRARIES})
+target_link_libraries(${LIB_CPROTO_NAME} ${Python3_LIBRARIES} ${REINDEXER_LIBRARIES})
+
+target_compile_definitions(${LIB_CPROTO_NAME} PRIVATE PYREINDEXER_CPROTO=1)
+
+set_target_properties(${LIB_BUILTIN_NAME} PROPERTIES SUFFIX ${LIBS_EXT})
+set_target_properties(${LIB_CPROTO_NAME} PROPERTIES SUFFIX ${LIBS_EXT})
+
+# python does not allow 'lib' prefix
+set_target_properties(${LIB_BUILTIN_NAME} PROPERTIES PREFIX "")
+set_target_properties(${LIB_CPROTO_NAME} PROPERTIES PREFIX "")
+
+# We do not want to strictly link to python libs,
+#   because we want to build wheels for all python versions using single machine / image / job.
+# When using linking there are unpleasant side effects, for example the wrong version of python is found or
+#   python is searched in the wrong place and is not found at all.
+# But we need Python.h during compilation and without this flag we will get "Undefined symbols" at the linking stage.
+# It's not the problem to skip this error, since the user will have his own python.
+if(CIBUILDWHEEL AND APPLE)
+    set_target_properties(${LIB_BUILTIN_NAME} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+    set_target_properties(${LIB_CPROTO_NAME} PROPERTIES LINK_FLAGS "-undefined dynamic_lookup")
+endif()
+
+if (WITH_GCOV)
+    target_link_libraries(${LIB_BUILTIN_NAME} -fprofile-arcs -ftest-coverage)
+    target_link_libraries(${LIB_CPROTO_NAME} -fprofile-arcs -ftest-coverage)
+endif (WITH_GCOV)

pyreindexer/__init__.py

@@ -0,0 +1,5 @@
+"""
+The pyreindexer module provides a connector and its auxiliary tools for interaction with Reindexer.
+"""
+
+from pyreindexer.rx_connector import RxConnector

pyreindexer/example/main.py

@@ -0,0 +1,315 @@
+import random
+
+from datetime import timedelta
+from typing import Final, List
+
+from pyreindexer import RxConnector
+from pyreindexer.index_search_params import IndexSearchParamHnsw
+from pyreindexer.exceptions import ApiError
+from pyreindexer.query import CondType
+
+
+def create_index_example(db, namespace):
+    index_definition = {
+        'name': 'id',
+        'json_paths': ['id'],
+        'field_type': 'int',
+        'index_type': 'hash',
+        'is_pk': True,
+        'is_array': False,
+        'is_dense': False,
+        'is_sparse': False,
+        'is_no_column': False,
+        'collate_mode': 'none',
+        'sort_order_letters': '',
+        'expire_after': 0,
+        'config': {},
+    }
+
+    try:
+        db.index_add(namespace, index_definition)
+    except ApiError:
+        db.index_drop(namespace, 'id', timedelta(milliseconds = 1000))
+        db.index_add(namespace, index_definition)
+
+
+def update_index_example(db, namespace):
+    index_definition_modified = {
+        'name': 'id',
+        'json_paths': ['id'],
+        'field_type': 'int64',
+        'index_type': 'hash',
+        'is_pk': True,
+        'is_array': False,
+        'is_dense': True,
+        'is_sparse': False,
+        'is_no_column': False,
+        'collate_mode': 'none',
+        'sort_order_letters': '',
+        'expire_after': 0,
+        'config': {},
+    }
+    db.index_update(namespace, index_definition_modified)
+
+
+def create_items_example(db, namespace):
+    items_count = 10
+
+    for i in range(0, items_count):
+        item = {'id': 1, 'name': 'item_' + str(i % 2), 'value': 'check'}
+        db.item_upsert(namespace, item, ["id=serial()"])
+
+
+def select_item_query_example(db, namespace):
+    item_name_for_lookup = 'item_0'
+
+    return db.exec_sql(f"SELECT * FROM {namespace} WHERE name='{item_name_for_lookup}'", timedelta(milliseconds = 1000))
+
+def select_all_item_query_example(db, namespace):
+    return db.exec_sql(f'SELECT * FROM {namespace}', timedelta(milliseconds = 1000))
+
+def print_all_records_from_namespace(db, namespace, message):
+    selected_items_tr = select_all_item_query_example(db, namespace)
+
+    res_count = selected_items_tr.count()
+    print(message, res_count)
+
+    for item in selected_items_tr:
+        print(f'item: {item}')
+
+def transaction_example(db, namespace, items_in_base):
+    # start transaction
+    transaction = db.new_transaction(namespace)
+
+    items_count = len(items_in_base)
+
+    # delete first few items
+    for i in range(int(items_count / 2)):
+        transaction.delete(items_in_base[i])
+
+    # update last one item, overwrite field 'value'
+    item = items_in_base[items_count - 1]
+    item['value'] = 'transaction was here'
+    transaction.update(item)
+
+    # stop transaction and commit changes to namespace
+    count = transaction.commit_with_count(timedelta(milliseconds = 1000))
+    print(f'Transaction updated count: {count}')
+
+    print_all_records_from_namespace(db, namespace, 'Transaction results count: ')
+
+def query_example(db, namespace):
+    # query all items
+    any_items = (db.new_query(namespace)
+                 .where('value', CondType.CondAny)
+                 .sort('id')
+                 .execute())
+    print(f'Query results count (Any): {any_items.count()}')
+    for item in any_items:
+        print(f'item: {item}')
+
+    # query some items
+    selected_items = (db.new_query(namespace)
+                      .where('value', CondType.CondEq, 'check')
+                      .sort('id')
+                      .limit(4)
+                      .execute(timedelta(milliseconds = 1000)))
+    print(f'Query results count (limited): {selected_items.count()}')
+    for item in selected_items:
+        print(f'item: {item}')
+
+    # delete some items
+    del_count = (db.new_query(namespace)
+                 .where('name', CondType.CondEq, 'item_1')
+                 .delete(timedelta(milliseconds = 1000)))
+    print(f'Deleted count: {del_count}')
+
+    # query all actual items
+    any_items = (db.new_query(namespace)
+                 .where('value', CondType.CondAny)
+                 .must_execute())
+    print(f'Query results count (Any after delete): {any_items.count()}')
+    for item in any_items:
+        print(f'item: {item}')
+
+def modify_query_transaction(db, namespace):
+    # start transaction
+    transaction = db.new_transaction(namespace)
+
+    # create an update query and set it for the transaction
+    query_upd = db.new_query(namespace).where("id", CondType.CondLe, 5).set("name", ["update_with_query_tx"])
+    transaction.update_query(query_upd)
+
+    # create a delete query and set it for the transaction
+    query_del = db.new_query(namespace).where("id", CondType.CondGe, 6)
+    transaction.delete_query(query_del)
+
+    # stop transaction and commit changes to namespace
+    transaction.commit(timedelta(milliseconds = 1000))
+
+    print_all_records_from_namespace(db, namespace, 'Transaction with Query results count: ')
+
+def random_vector(dimension: int) -> List[float]:
+    return [random.uniform(-10.0, 10.0) for _ in range(dimension)]
+
+def float_vector_hnsw_example(db):
+    namespace = 'knn_hnsw'
+    db.namespace_open(namespace)
+
+    # create index
+    fv_index_name = 'hnsw_idx'
+    dimension: Final[int] = 4
+    index_definitions = [{'name': 'id',
+                          'json_paths': ['id'],
+                          'field_type': 'int',
+                          'index_type': 'hash',
+                          'is_pk': True,
+                          'is_array': False,
+                          'is_dense': False,
+                          'is_sparse': False,
+                          'is_no_column': False,
+                          'collate_mode': 'none',
+                          'sort_order_letters': '',
+                          'expire_after': 0,
+                          'config': {}},
+                         {"name": fv_index_name,
+                          "json_paths": [fv_index_name],
+                          "field_type": "float_vector",
+                          "index_type": "hnsw",
+                          "config": {
+                              "dimension": dimension,
+                              "metric": "inner_product",
+                              "start_size": 100,
+                              "m": 16,
+                              "ef_construction": 200,
+                              "multithreading": 1}}]
+    for index in index_definitions:
+        db.index_add(namespace, index)
+
+    # generate items
+    transaction = db.new_transaction(namespace)
+    for i in range(100):
+        transaction.insert({"id": 0, fv_index_name: random_vector(dimension)}, ["id=serial()"])
+    transaction.commit(timedelta(seconds = 3))
+
+    # do query
+    param = IndexSearchParamHnsw(k=20, ef=30)
+    query_result = (db.new_query(namespace)
+                        .where_knn(fv_index_name, random_vector(dimension), param)
+                        .select_fields("vectors()")
+                        .with_rank()
+                        .sort(index="rank()", desc=True)
+                        .must_execute(timedelta(seconds = 1)))
+
+    # result
+    print("HNSW where_knn: ", query_result.count())
+    for item in query_result:
+        print('item vec: ', item, end='\n')
+
+    # drop index
+    db.index_drop(namespace, fv_index_name, timedelta(milliseconds = 300))
+
+def float_vector_brute_force_sql_example(db):
+    namespace = 'knn_bf'
+    db.namespace_open(namespace)
+
+    # create index
+    fv_index_name = 'bf_idx'
+    dimension: Final[int] = 4
+    index_definitions = [{'name': 'id',
+                          'json_paths': ['id'],
+                          'field_type': 'int',
+                          'index_type': 'hash',
+                          'is_pk': True,
+                          'is_array': False,
+                          'is_dense': False,
+                          'is_sparse': False,
+                          'is_no_column': False,
+                          'collate_mode': 'none',
+                          'sort_order_letters': '',
+                          'expire_after': 0,
+                          'config': {}},
+                         {"name": fv_index_name,
+                          "json_paths": [fv_index_name],
+                          "field_type": "float_vector",
+                          "index_type": "vec_bf",
+                          "config": {
+                              "dimension": 3,
+                              "metric": "inner_product",
+                              "start_size": 10000}}]
+    for index in index_definitions:
+        db.index_add(namespace, index)
+
+    # update index
+    index_definition_modified = {"name": fv_index_name,
+                                 "json_paths": [fv_index_name],
+                                 "field_type": "float_vector",
+                                 "index_type": "vec_bf",
+                                 "config": {
+                                     "dimension": dimension,
+                                     "metric": "l2",
+                                     "start_size": 1000}}
+    db.index_update(namespace, index_definition_modified)
+
+    # generate items
+    transaction = db.new_transaction(namespace)
+    for i in range(100):
+        transaction.insert({"id": i, fv_index_name: random_vector(dimension)})
+    transaction.commit(timedelta(seconds = 3))
+
+    # execute SQL query SELECT KNN
+    value = random_vector(dimension)
+    k: Final[int] = 27
+    query = f'SELECT *, vectors() FROM {namespace} WHERE KNN({fv_index_name}, {value}, k={k})'
+    query_result = db.exec_sql(query, timedelta(seconds = 1))
+    print("Select where KNN: ", query_result.count())
+    for item in query_result:
+        print('item vec: ', item, end='\n')
+
+    # drop index
+    db.index_drop(namespace, fv_index_name, timedelta(milliseconds = 300))
+
+
+def rx_example():
+
+    db = RxConnector(f'builtin:///tmp/pyrx', max_replication_updates_size = 10 * 1024 * 1024)
+    #db = RxConnector('cproto://127.0.0.1:6534/pyrx', enable_compression = True, fetch_amount = 500)
+
+    namespace = 'test_table'
+    db.namespace_open(namespace)
+
+    create_index_example(db, namespace)
+    update_index_example(db, namespace)
+
+    create_items_example(db, namespace)
+    print_all_records_from_namespace(db, namespace, 'All items: ')
+
+    selected_items = select_item_query_example(db, namespace)
+
+    res_count = selected_items.count()
+    print(f'Results count: {res_count}')
+
+    # disposable QueryResults iterator
+    items_copy = []
+    for item in selected_items:
+        items_copy.append(item)
+        print(f'item: {item}')
+
+    # won't be iterated again
+    for item in selected_items:
+        print(f'item: {item}')
+
+    transaction_example(db, namespace, items_copy)
+
+    query_example(db, namespace)
+
+    modify_query_transaction(db, namespace)
+
+    float_vector_hnsw_example(db)
+    float_vector_brute_force_sql_example(db)
+
+    db.close()
+
+
+if __name__ == "__main__":
+    rx_example()

pyreindexer/exceptions.py

@@ -0,0 +1,10 @@
+class ApiError(Exception):
+    pass
+
+
+class QueryError(ApiError):
+    pass
+
+
+class TransactionError(ApiError):
+    pass

pyreindexer/index_definition.py

@@ -0,0 +1,58 @@
+# TODO NOT IMPLEMENTED YET
+# TODO dynamic setters which return self. e.g.: indexDef.name('test_name').is_pk().is_dense()
+# TODO check types for each attrs of index definition
+# TODO check possible values for attrs field_type, index_type, collate_mode
+
+
+class IndexDefinition(dict):
+    """IndexDefinition is a dictionary subclass which allows to construct and manage indexes more efficiently.
+    NOT IMPLEMENTED YET. USE FIELDS DESCRIPTION ONLY.
+
+    #### Arguments:
+        name (str): An index name.
+        json_paths (:obj:`list` of :obj:`str`): A name for mapping a value to a json field.
+        field_type (str): A type of field. Possible values are: `int`, `int64`, `double`, `string`, `bool`,
+        `composite`, `float_vector`.
+        index_type (str): An index type. Possible values are: `hash`, `tree`, `text`, `-`, `hnsw`, `vec_bf`, `ivf`.
+        is_pk (bool): True if a field is a primary key.
+        is_array (bool): True if an index is an array.
+        is_dense (bool): True if an index is dense. Reduce the index size. Saves 8 bytes per unique key value for 'hash'
+            and 'tree' index types. For '-' index type saves 4-8 bytes per each element. Useful for indexes with
+            high selectivity, but for tree and hash indexes with low selectivity can seriously decrease update
+            performance.
+        is_no_column (bool): True if allows to disable column subindex. Reduces the index size.
+            Allows to save ~(`stored_type_size` * `namespace_items_count`) bytes, where `stored_type_size` is the size
+            of the type stored in the index, and `namespace_items_count` is the number of items in the namespace.
+            May reduce performance.
+        is_sparse (bool): True if a value of an index may be not presented.
+        collate_mode (str): Sets an order of values by collate mode. Possible values are:
+            `none`, `ascii`, `utf8`, `numeric`, `custom`.
+        sort_order_letters (str): Order for a sort sequence for a custom collate mode.
+        config (dict): A config for a fulltext and float_vector engine.
+        [More about `fulltext`](https://github.com/Restream/reindexer/blob/master/fulltext.md) or
+        [More about `float_vector`](https://github.com/Restream/reindexer/blob/master/float_vector.md).
+    """
+
+    def __getitem__(self, attr):
+        self._raise_if_key_error(attr)
+        return super(IndexDefinition, self).get(attr)
+
+    def __setitem__(self, attr, value):
+        self._raise_if_key_error(attr)
+        super(IndexDefinition, self).update({attr: value})
+        return self
+
+    def update(self, *args, **kwargs):
+        raise NotImplementedError(
+            'Bulk update is not implemented for IndexDefinition instance')
+
+    @staticmethod
+    def _get_known_attrs() -> list[str]:
+        return ['name', 'json_paths', 'field_type', 'index_type', 'is_pk',
+                'is_array', 'is_dense', 'is_sparse', 'collate_mode', 'sort_order_letters', 'expire_after', 'config']
+
+    def _raise_if_key_error(self, attr):
+        known_attrs = self._get_known_attrs()
+        if attr not in known_attrs:
+            raise KeyError("Invalid key '{0}'. Known keys are: '{1}'".format(
+                attr, ', '.join(known_attrs)))

pyreindexer/index_search_params.py

@@ -0,0 +1,68 @@
+class IndexSearchParamBruteForce:
+    """Index search param for brute force index. Equal to basic parameters
+
+    #### Attributes:
+        k (int): Expected size of KNN index results. Should not be less than 1
+        radius (float): In addition to the parameter `k`, the query results can also be filtered by a `rank` -
+            value using the parameter, witch called `radius`. It's named so because, under the `L2`-metric,
+            it restricts vectors from query result to a sphere of the specified radius
+
+    """
+
+    def __init__(self, k: int = None, radius: float = None):
+        if k is None and radius is None:
+            raise ValueError("Either 'k' or 'radius' needs to be specified")
+        if k is not None and k < 1:
+            raise ValueError("KNN limit 'k' should not be less than 1")
+        self.k = k
+        self.radius = radius
+
+
+class IndexSearchParamHnsw:
+    """Index search param for HNSW index.
+
+    #### Attributes:
+        k (int): Expected size of KNN index results. Should not be less than 1
+        ef (int): Size of nearest neighbor buffer that will be filled during fetching. Should not be less than 'k',
+            good story when `ef` ~= 1.5 * `k`
+        radius (float): In addition to the parameter `k`, the query results can also be filtered by a `rank` -
+            value using the parameter, witch called `radius`. It's named so because, under the `L2`-metric,
+            it restricts vectors from query result to a sphere of the specified radius
+
+    """
+
+    def __init__(self, ef: int, k: int = None, radius: float = None):
+        if k is None and radius is None:
+            raise ValueError("Either 'k' or 'radius' needs to be specified")
+        if k is not None:
+            if k < 1:
+                raise ValueError("KNN limit 'k' should not be less than 1")
+            if ef < k:
+                raise ValueError("'ef' should not be less than 'k'")
+        self.k = k
+        self.radius = radius
+        self.ef = ef
+
+
+class IndexSearchParamIvf:
+    """Index search param for IVF index.
+
+    #### Attributes:
+        k (int): Expected size of KNN index results. Should not be less than 1
+        nprobe (int): Number of centroids that will be scanned in where. Should not be less than 1
+        radius (float): In addition to the parameter `k`, the query results can also be filtered by a `rank` -
+            value using the parameter, witch called `radius`. It's named so because, under the `L2`-metric,
+            it restricts vectors from query result to a sphere of the specified radius
+
+    """
+
+    def __init__(self, nprobe: int, k: int = None, radius: float = None):
+        if k is None and radius is None:
+            raise ValueError("Either 'k' or 'radius' needs to be specified")
+        if k is not None and k < 1:
+            raise ValueError("KNN limit 'k' should not be less than 1")
+        if nprobe < 1:
+            raise ValueError("'nprobe' should not be less than 1")
+        self.nprobe = nprobe
+        self.k = k
+        self.radius = radius