nidx-binding 6.6.1.post444__tar.gz → 6.6.1.post454__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/PKG-INFO +1 -1
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/pyproject.toml +1 -1
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v1/store.rs +1 -1
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v1.rs +1 -1
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v2/paragraph_store.rs +1 -1
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v2.rs +1 -1
- {nidx_binding-6.6.1.post444/nidx_vector/src/data_point → nidx_binding-6.6.1.post454/nidx_vector/src/hnsw}/disk_hnsw.rs +1 -1
- {nidx_binding-6.6.1.post444/nidx_vector/src/data_point → nidx_binding-6.6.1.post454/nidx_vector/src/hnsw}/ops_hnsw.rs +4 -1
- nidx_binding-6.6.1.post454/nidx_vector/src/hnsw.rs +48 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/indexer.rs +3 -3
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/lib.rs +19 -31
- nidx_binding-6.6.1.post444/nidx_vector/src/data_point_provider/reader.rs → nidx_binding-6.6.1.post454/nidx_vector/src/searcher.rs +72 -69
- {nidx_binding-6.6.1.post444/nidx_vector/src/data_point → nidx_binding-6.6.1.post454/nidx_vector/src/segment}/tests.rs +42 -42
- nidx_binding-6.6.1.post444/nidx_vector/src/data_point/mod.rs → nidx_binding-6.6.1.post454/nidx_vector/src/segment.rs +30 -42
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/tests/test_basic_search.rs +12 -12
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/tests/test_hidden.rs +3 -3
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/tests/test_maxsim.rs +4 -4
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/pyproject.toml +1 -1
- nidx_binding-6.6.1.post444/nidx_vector/src/data_point_provider/mod.rs +0 -32
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.config/nextest.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-0cfce9b29547f8f5bafa6e440f86103be7b8c4ad2fd92db9ac223f4efbe23d10.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-1a561eed00f3dbe868bf5030059793300209179dc8fb73e4b57a54b5e81262fe.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-1d3fca2682e25a01143da92285297f134a6a105a96f64d87e0db3abb219855e4.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-249b3b57c27a71baa823f1fe0f0bba9c9af36f61c28f731e58beea60ec48e687.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-24cb6b683daa42d7125f862e25943ab4be7bf275cd8739f8da4859d701795e1a.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-263c8fce6db5b03bbd012fafdba6943cbee6ed7eb8976cdef4f5b01dde7ca6fd.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-2a5d92fb1638df830a4477a7cdf24e6db6b43034b7bbe74fdfb63e8afe2c4071.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-2b065a363f58caed60e3706603c1260dbf5a4c795604a5b68edda22eb07fec1b.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-3fc3cb39934683de8cd475ce1368c8373453eb1e01f81587d66b9d14b109ce6e.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-48f33b77b7c1633467b0b2efcaa1d3c207e7757e4f1d83b40d15e6ca365f7771.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-4ae09f2c08e2f324bee01bb8487a8f37678a1c5e9d327339235c50d4921a8949.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-4d7a76fa413c9ef0ce2a47ac7bb7e01d3e6a2aabded9487d21010a53efee8852.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-4fcbdd6657c7dc9b60b3a563dd41711b3dbcf72ce063427b7a01f8cddf34c244.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-577109ac00ccfbd38ecaccab94116f2f46a4caf5612afa372cded197123c1e08.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-5db25f97d8578d6d78f2f6bd4b72cc82a9b1b82805c6422d967ac63b20d99db4.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-5ec3233a3a23e926055056d46bdde17836a633066dbb5f349502648cd3ea9a60.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-66edb6ea424d8681927dcddb6bac5f1239175f4775d1f40417ba15054b0c6f19.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-6f9c6d201c1b5712efb68c363bffd3e0169c11f2a8f925e8cd4e8808599ff7b4.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-733c3ebacc86f444bf5e2dd79ade660c291e88a00fc09b722f6e2e191545874c.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-7a3bf27c330c468a596e8a297cf7d8b192e31e67ecc5177c1267f579e8e247c7.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-7a7e59e47b30b12237511fd3d7da2d17b0471ad2b006af48d6a6f587c779692b.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-7dcbb33312cc9f11ae3a6d73b1ace017a9f19a8bf8f10304fc57977c8efeadff.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-7efa7c0d747afc4b6aed0586ff846c27839c3213ff7ee9f30c89b0d0f17e60e3.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-8493140d788604d498a4e48da4158708572ccc9d60185290a00d549cc84533db.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-8493bb0059b013eaca42fd10cd7d04f0d06a8acaed379eff0d23f3229edde9ee.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-87996b3d6c7a2195438d7038015b06949102bce8c7b8cd8db1f83aaf23cbe489.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-8d33717587c6ee8f5fc339a80b1212a73d6c03e45856b1d55457fc8074709dd0.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-8f096d8171b89f9615d18f95d696dc9e4fb3674e103161a713cdc806f7a68506.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-917732a56ee04bf3a6e127319dda8225210869c82f9828d878162394dba4e078.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-95fe4ef93ee90733db1b67ed7987f80b5aac792f1590b979c68b418d1599eb98.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-9b67658569b343d8b4b61ae0a7dc721f367f2ba33c7b69b9e68bfd5c9bff5206.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-9c8062ea55d070afef68309e58fa987eb37fda44e1efbf68c8ba2af7846cc968.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-a06e1d9f6f95e4c4c2b98310ebddcc9d963cc033582bf2e945e8bf3a301b4247.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-a55265c9b07bd1399961a6f1e757201fd0eebe868ddaf96437111113d80fce92.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-a60ec2f66f1e7b84189e5b089f2087a29ff6a64326a3743dea935bbc58ee77fa.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-a891a37be5c2d7cce775c2dd33726b0318fd3839beab222a1b22bc6174604207.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-a945191bb4b3e37d6823ed3ad499339d007d69983105de8567777d9daf517b28.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-abe9f7832f2bd799ac44008da031e8d8ab52d4f5fbfc2a7e3974e8873bae55b2.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-aca588cca57a85e4d7fcc40c23cd87e57d53d11ca550d78e7e3d5e39e524fcd3.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-b02f8aafc00a7724510772ac41269e368c5bccf03ef7b4590e0ef6fd1a1bf64f.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-b742e17cabe2d64617e9aa64bafc782172f7a4f8023d1b54f952a0fb39f6b2b8.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-b94e349dbc0daec57f8f8f6e9e2dffb06100b1bb2b41d297c9f3b191da37a83d.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-bd9afa22994aba671dbf7b5f89b53c2ee02f53c0442a81265786a6d52d08512f.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-be60554eca98a5899efc6b49785cecd6444a6d39afed9e4a884ce2dbf162012c.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-bf49702b506c9a1650ece1f8e8d9f14834a902f8caefafe30ded55e2790f2188.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-bfcd21ed704cd305db5c17fcdec7d92aa4ac501913c9c9514d8ff92928c0c7e7.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-c3ab694650f49a75b146fb877a92e48c4f20f0d99f70f8ec859fbb763b01a1e5.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-c55542bb9fae544d87fae6f30e0fe8a9088d12075f4442ab4fe2fcd05e472234.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-cb29a6556d35ac630ee0aa885dd7341cf9573bd3efd216ff8a887b87686b03db.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-d0a1f341a89f5f14696b10baa72db9d95551c2b7e5fc67308fd52dc03dd98a92.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-d2ad0a0ca2649c9e4873cfcc1fc66d2d07cc45d0f65c560b06d7b5f592f4fa8a.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-d6cfe78eb635ba0b89ca4021a4dc8182d18ab5b197f30149cd28488eba4c1df5.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-d729b56dea00e49dcdba8cf0001e2811da27351eabe98212db3b589f18fc6f32.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-d9658bfd4e7170b41d03f2ddf2446d0bf54171c0d39d53bf20af2b8437f2ec48.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-dbba7b3d3289425bae711aedbf73fbc3699f857f86f84d95c3b556d05c5658b0.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-dcb96b649d6d63a58efd5d445453a4f3d7869a56ff714b69bedf3d616a0473ca.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-ebd876fbf5362a5900e75bc05f2f11c73c406ef7da4e95097fc6a1c3d1b8bc54.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-eef5cc6bce1cc14eba8f3e68971724ef181e88cffcedd74673615f2026b89a62.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/.sqlx/query-ef56d5fefc5774040d1ee397beadb475f6af02768c22f0e583c74062e2e821ce.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/Cargo.lock +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/README.md +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/migrations/20241007163501_initial.sql +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/migrations/20241211120039_merge_job_priority.sql +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/migrations/20241211121159_basic_indexes.sql +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/migrations/20241212151105_check_segment_records.sql +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/migrations/20250110145554_in_flight_messages.sql +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_binding/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_binding/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/fuzzy_query.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/query_io.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/reader.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/request_types.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/resource_indexer.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/schema.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/search_query.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/search_response.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/set_query.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/src/stop_words.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/README.md +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/ar.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/az.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/bn.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/ca.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/ch.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/da.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/de.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/el.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/en.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/es.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/eu.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/extract.py +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/fi.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/fr.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/he.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/hu.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/id.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/it.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/kk.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/ne.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/nl.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/no.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/pt.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/ro.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/ru.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/sl.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/sv.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/tg.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/stop_words/tr.json +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/tests/common/mod.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_paragraph/tests/reader.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/build.py +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/build.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/nidx.proto +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/nodereader.proto +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/noderesources.proto +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/nodewriter.proto +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_protos/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/graph_collector.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/graph_query_parser.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/io_maps.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/reader.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/resource_indexer.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/schema.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/src/top_unique_n.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/tests/common/mod.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/tests/test_graph_query_parser_search.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/tests/test_graph_search.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_relation/tests/test_writer.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_tantivy/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_tantivy/src/index_reader.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_tantivy/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_tantivy/src/utils.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_tests/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_tests/src/graph.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_tests/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/prefilter.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/query_io.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/reader.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/request_types.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/resource_indexer.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/schema.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/src/search_query.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/tests/common/mod.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/tests/test_deletions.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/tests/test_flow.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/tests/test_search.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_text/tests/test_streaming.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_types/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_types/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_types/src/prefilter.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_types/src/query_language.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/Cargo.toml +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/config.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v1/node.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v1/trie.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v1/trie_ram.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v2/vector_store.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_types.rs +0 -0
- /nidx_binding-6.6.1.post444/nidx_vector/src/formula/mod.rs → /nidx_binding-6.6.1.post454/nidx_vector/src/formula.rs +0 -0
- {nidx_binding-6.6.1.post444/nidx_vector/src/data_point → nidx_binding-6.6.1.post454/nidx_vector/src/hnsw}/params.rs +0 -0
- {nidx_binding-6.6.1.post444/nidx_vector/src/data_point → nidx_binding-6.6.1.post454/nidx_vector/src/hnsw}/ram_hnsw.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/inverted_index/fst_index.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/inverted_index/map.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/inverted_index.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/multivector.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/query_io.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/request_types.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/utils.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/vector_types/dense_f32.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/vector_types/mod.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/tests/common/mod.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/api/grpc.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/api/shards.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/api.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/control.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/errors.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/grpc_server.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/import_export.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/indexer.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/lib.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/main.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metadata/deletion.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metadata/index.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metadata/index_request.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metadata/merge_job.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metadata/segment.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metadata/shard.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metadata.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/metrics.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/scheduler/audit_task.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/scheduler/log_merge.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/scheduler/merge_task.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/scheduler/metrics_task.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/scheduler/purge_tasks.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/scheduler/vector_merge.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/scheduler.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/grpc.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/index_cache.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/query_language.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/query_planner.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/shard_search.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/shard_selector.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/shard_suggest.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/streams.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher/sync.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/searcher.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/segment_store.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/settings.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/telemetry/duration_layer.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/telemetry/log_format.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/telemetry/middleware.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/telemetry.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/tool.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/utilization_tracker.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/src/worker.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/common/mod.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/common/services.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_date_range_search.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_search_filtering.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_search_relations.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_search_sorting.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_searcher_cluster.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_security_search.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_shards.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_shards_api.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_suggest.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_synced_searcher.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_vector_normalization.rs +0 -0
- {nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/tests/test_vectorsets.rs +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.4
|
2
2
|
Name: nidx_binding
|
3
|
-
Version: 6.6.1.
|
3
|
+
Version: 6.6.1.post454
|
4
4
|
Classifier: Programming Language :: Rust
|
5
5
|
Classifier: Programming Language :: Python :: Implementation :: CPython
|
6
6
|
Classifier: Programming Language :: Python :: Implementation :: PyPy
|
@@ -10,7 +10,7 @@ build-backend = "pdm.backend"
|
|
10
10
|
|
11
11
|
[project]
|
12
12
|
name = "nidx_protos"
|
13
|
-
version = "6.6.1.
|
13
|
+
version = "6.6.1.post454"
|
14
14
|
license = "AGPL-3.0-or-later"
|
15
15
|
description = "Protobuf definitions for nucliadb/nidx"
|
16
16
|
authors = [{ name = "Nuclia", email = "nucliadb@nuclia.com" }]
|
{nidx_binding-6.6.1.post444 → nidx_binding-6.6.1.post454}/nidx_vector/src/data_store/v1/store.rs
RENAMED
@@ -211,7 +211,7 @@ pub fn merge(
|
|
211
211
|
#[cfg(test)]
|
212
212
|
mod tests {
|
213
213
|
use super::*;
|
214
|
-
use crate::{config::VectorCardinality,
|
214
|
+
use crate::{config::VectorCardinality, segment::Elem};
|
215
215
|
|
216
216
|
const VECTOR_CONFIG: VectorConfig = VectorConfig {
|
217
217
|
vector_type: VectorType::DenseF32 { dimension: 3 },
|
@@ -23,9 +23,9 @@ use std::{fs::File, io::Write as _, path::Path};
|
|
23
23
|
|
24
24
|
use crate::{
|
25
25
|
VectorR,
|
26
|
-
data_point::Elem,
|
27
26
|
data_store::{OpenReason, ParagraphAddr, ParagraphRef},
|
28
27
|
data_types::usize_utils::U32_LEN,
|
28
|
+
segment::Elem,
|
29
29
|
};
|
30
30
|
|
31
31
|
const FILENAME_DATA: &str = "paragraphs.bin";
|
@@ -18,7 +18,7 @@
|
|
18
18
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
19
|
//
|
20
20
|
|
21
|
-
use crate::{VectorR, config::VectorType,
|
21
|
+
use crate::{VectorR, config::VectorType, segment::Elem};
|
22
22
|
|
23
23
|
use super::{DataStore, OpenReason, ParagraphAddr, VectorAddr};
|
24
24
|
pub use paragraph_store::StoredParagraph;
|
@@ -260,7 +260,7 @@ impl DiskHnsw {
|
|
260
260
|
#[cfg(test)]
|
261
261
|
mod tests {
|
262
262
|
use super::*;
|
263
|
-
use crate::
|
263
|
+
use crate::hnsw::ram_hnsw::RAMLayer;
|
264
264
|
fn layer_check<L: Layer>(buf: L, no_nodes: usize, cnx: &[Vec<(Address, Edge)>]) {
|
265
265
|
let no_cnx = vec![];
|
266
266
|
for i in 0..no_nodes {
|
@@ -27,8 +27,11 @@ use rand::rngs::SmallRng;
|
|
27
27
|
use std::cmp::{Ordering, Reverse};
|
28
28
|
use std::collections::{BinaryHeap, HashMap, HashSet, VecDeque};
|
29
29
|
|
30
|
+
use crate::ParagraphAddr;
|
31
|
+
use crate::inverted_index::FilterBitSet;
|
32
|
+
|
33
|
+
use super::params;
|
30
34
|
use super::*;
|
31
|
-
use crate::data_point::params;
|
32
35
|
|
33
36
|
/// Implementors of this trait can guide the hnsw search
|
34
37
|
pub trait DataRetriever: std::marker::Sync {
|
@@ -0,0 +1,48 @@
|
|
1
|
+
// Copyright (C) 2021 Bosutech XXI S.L.
|
2
|
+
//
|
3
|
+
// nucliadb is offered under the AGPL v3.0 and as commercial software.
|
4
|
+
// For commercial licensing, contact us at info@nuclia.com.
|
5
|
+
//
|
6
|
+
// AGPL:
|
7
|
+
// This program is free software: you can redistribute it and/or modify
|
8
|
+
// it under the terms of the GNU Affero General Public License as
|
9
|
+
// published by the Free Software Foundation, either version 3 of the
|
10
|
+
// License, or (at your option) any later version.
|
11
|
+
//
|
12
|
+
// This program is distributed in the hope that it will be useful,
|
13
|
+
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
14
|
+
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
15
|
+
// GNU Affero General Public License for more details.
|
16
|
+
//
|
17
|
+
// You should have received a copy of the GNU Affero General Public License
|
18
|
+
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
|
+
//
|
20
|
+
|
21
|
+
mod disk_hnsw;
|
22
|
+
mod ops_hnsw;
|
23
|
+
mod params;
|
24
|
+
mod ram_hnsw;
|
25
|
+
|
26
|
+
pub use disk_hnsw::DiskHnsw;
|
27
|
+
pub use ops_hnsw::Cnx;
|
28
|
+
pub use ops_hnsw::DataRetriever;
|
29
|
+
pub use ops_hnsw::HnswOps;
|
30
|
+
pub use ram_hnsw::RAMHnsw;
|
31
|
+
|
32
|
+
use crate::VectorAddr;
|
33
|
+
|
34
|
+
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Default)]
|
35
|
+
pub struct Address(pub(super) usize);
|
36
|
+
|
37
|
+
// VectorAddr and HNSW Address are the same thing with a different data type for serialization purposes
|
38
|
+
impl From<Address> for VectorAddr {
|
39
|
+
fn from(value: Address) -> Self {
|
40
|
+
Self(value.0 as u32)
|
41
|
+
}
|
42
|
+
}
|
43
|
+
|
44
|
+
impl From<VectorAddr> for Address {
|
45
|
+
fn from(value: VectorAddr) -> Self {
|
46
|
+
Self(value.0 as usize)
|
47
|
+
}
|
48
|
+
}
|
@@ -18,8 +18,8 @@
|
|
18
18
|
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
19
19
|
|
20
20
|
use crate::config::{VectorCardinality, VectorConfig};
|
21
|
-
use crate::data_point::{self, Elem};
|
22
21
|
use crate::multivector::extract_multi_vectors;
|
22
|
+
use crate::segment::{self, Elem};
|
23
23
|
use crate::{VectorSegmentMetadata, utils};
|
24
24
|
use nidx_protos::{noderesources, prost::*};
|
25
25
|
use std::collections::HashMap;
|
@@ -157,10 +157,10 @@ pub fn index_resource(
|
|
157
157
|
.filter(|t| SEGMENT_TAGS.contains(&t.as_str()))
|
158
158
|
.cloned()
|
159
159
|
.collect();
|
160
|
-
let
|
160
|
+
let segment = segment::create(output_path, elems, config, tags)?;
|
161
161
|
|
162
162
|
let v = time.elapsed().as_millis();
|
163
163
|
debug!("{id:?} - Main index set resource: ends {v} ms");
|
164
164
|
|
165
|
-
Ok(Some(
|
165
|
+
Ok(Some(segment.into_metadata()))
|
166
166
|
}
|
@@ -19,26 +19,27 @@
|
|
19
19
|
//
|
20
20
|
|
21
21
|
pub mod config;
|
22
|
-
mod data_point;
|
23
|
-
mod data_point_provider;
|
24
22
|
mod data_store;
|
25
23
|
mod data_types;
|
26
24
|
mod formula;
|
25
|
+
mod hnsw;
|
27
26
|
mod indexer;
|
28
27
|
mod inverted_index;
|
29
28
|
mod multivector;
|
30
29
|
mod query_io;
|
31
30
|
mod request_types;
|
31
|
+
mod searcher;
|
32
|
+
mod segment;
|
32
33
|
mod utils;
|
33
34
|
mod vector_types;
|
34
35
|
|
35
36
|
use config::VectorConfig;
|
36
|
-
use data_point::{Address, OpenDataPoint};
|
37
|
-
use data_point_provider::reader::Reader;
|
38
37
|
use indexer::{ResourceWrapper, index_resource};
|
39
38
|
use nidx_protos::{Resource, VectorSearchResponse};
|
40
39
|
use nidx_types::prefilter::PrefilterResult;
|
41
40
|
use nidx_types::{OpenIndexMetadata, SegmentMetadata};
|
41
|
+
use searcher::Searcher;
|
42
|
+
use segment::OpenSegment;
|
42
43
|
use serde::{Deserialize, Serialize};
|
43
44
|
use std::collections::HashSet;
|
44
45
|
use std::path::Path;
|
@@ -52,19 +53,6 @@ pub use request_types::VectorSearchRequest;
|
|
52
53
|
pub struct ParagraphAddr(u32);
|
53
54
|
pub struct VectorAddr(u32);
|
54
55
|
|
55
|
-
// VectorAddr and HNSW Address are the same thing with a different data type for serialization purposes
|
56
|
-
impl From<Address> for VectorAddr {
|
57
|
-
fn from(value: Address) -> Self {
|
58
|
-
Self(value.0 as u32)
|
59
|
-
}
|
60
|
-
}
|
61
|
-
|
62
|
-
impl From<VectorAddr> for Address {
|
63
|
-
fn from(value: VectorAddr) -> Self {
|
64
|
-
Self(value.0 as usize)
|
65
|
-
}
|
66
|
-
}
|
67
|
-
|
68
56
|
#[derive(Clone, Serialize, Deserialize)]
|
69
57
|
pub struct VectorSegmentMeta {
|
70
58
|
pub tags: HashSet<String>,
|
@@ -105,25 +93,25 @@ impl VectorIndexer {
|
|
105
93
|
config: VectorConfig,
|
106
94
|
open_index: impl OpenIndexMetadata<VectorSegmentMeta>,
|
107
95
|
) -> anyhow::Result<VectorSegmentMetadata> {
|
108
|
-
let
|
109
|
-
let
|
96
|
+
let open_segments = open_segments(open_index, &config)?;
|
97
|
+
let open_segments_ref = open_segments.iter().collect::<Vec<_>>();
|
110
98
|
|
111
99
|
// Do the merge
|
112
|
-
let open_destination =
|
100
|
+
let open_destination = segment::merge(work_dir, &open_segments_ref, &config)?;
|
113
101
|
|
114
102
|
Ok(open_destination.into_metadata())
|
115
103
|
}
|
116
104
|
}
|
117
105
|
|
118
106
|
pub struct VectorSearcher {
|
119
|
-
|
107
|
+
searcher: Searcher,
|
120
108
|
}
|
121
109
|
|
122
110
|
impl VectorSearcher {
|
123
111
|
#[instrument(name = "vector::open", skip_all)]
|
124
112
|
pub fn open(config: VectorConfig, open_index: impl OpenIndexMetadata<VectorSegmentMeta>) -> anyhow::Result<Self> {
|
125
113
|
Ok(VectorSearcher {
|
126
|
-
|
114
|
+
searcher: Searcher::open(open_segments(open_index, &config)?, config)?,
|
127
115
|
})
|
128
116
|
}
|
129
117
|
|
@@ -133,35 +121,35 @@ impl VectorSearcher {
|
|
133
121
|
request: &VectorSearchRequest,
|
134
122
|
prefilter: &PrefilterResult,
|
135
123
|
) -> anyhow::Result<VectorSearchResponse> {
|
136
|
-
self.
|
124
|
+
self.searcher.search(request, prefilter)
|
137
125
|
}
|
138
126
|
|
139
127
|
pub fn space_usage(&self) -> usize {
|
140
|
-
self.
|
128
|
+
self.searcher.space_usage()
|
141
129
|
}
|
142
130
|
}
|
143
131
|
|
144
132
|
fn open_segments(
|
145
133
|
open_index: impl OpenIndexMetadata<VectorSegmentMeta>,
|
146
134
|
config: &VectorConfig,
|
147
|
-
) -> VectorR<Vec<
|
148
|
-
let mut
|
135
|
+
) -> VectorR<Vec<OpenSegment>> {
|
136
|
+
let mut open_segments = Vec::new();
|
149
137
|
|
150
138
|
for (metadata, seq) in open_index.segments() {
|
151
|
-
let
|
139
|
+
let open_segment = segment::open(metadata, config)?;
|
152
140
|
|
153
|
-
|
141
|
+
open_segments.push((open_segment, seq));
|
154
142
|
}
|
155
143
|
|
156
144
|
for (deletion, deletion_seq) in open_index.deletions() {
|
157
|
-
for (segment, segment_seq) in &mut
|
145
|
+
for (segment, segment_seq) in &mut open_segments {
|
158
146
|
if deletion_seq > *segment_seq {
|
159
147
|
segment.apply_deletion(deletion.as_str());
|
160
148
|
}
|
161
149
|
}
|
162
150
|
}
|
163
151
|
|
164
|
-
Ok(
|
152
|
+
Ok(open_segments.into_iter().map(|(dp, _)| dp).collect())
|
165
153
|
}
|
166
154
|
|
167
155
|
#[derive(Debug, Error)]
|
@@ -176,7 +164,7 @@ pub enum VectorErr {
|
|
176
164
|
UncommittedChangesError,
|
177
165
|
#[error("Merger is already initialized")]
|
178
166
|
MergerAlreadyInitialized,
|
179
|
-
#[error("Can not merge zero
|
167
|
+
#[error("Can not merge zero segments")]
|
180
168
|
EmptyMerge,
|
181
169
|
#[error("Inconsistent dimensions. Index={index_config} Vector={vector}")]
|
182
170
|
InconsistentDimensions { index_config: usize, vector: usize },
|
@@ -20,13 +20,12 @@
|
|
20
20
|
|
21
21
|
use crate::ParagraphAddr;
|
22
22
|
use crate::config::VectorCardinality;
|
23
|
-
use crate::
|
24
|
-
use crate::data_point_provider::SearchRequest;
|
25
|
-
use crate::data_point_provider::VectorConfig;
|
23
|
+
use crate::config::VectorConfig;
|
26
24
|
use crate::data_store::ParagraphRef;
|
27
25
|
use crate::multivector::extract_multi_vectors;
|
28
26
|
use crate::multivector::maxsim_similarity;
|
29
27
|
use crate::request_types::VectorSearchRequest;
|
28
|
+
use crate::segment::OpenSegment;
|
30
29
|
use crate::utils;
|
31
30
|
use crate::{VectorErr, VectorR};
|
32
31
|
use crate::{formula::*, query_io};
|
@@ -40,12 +39,35 @@ use std::collections::HashSet;
|
|
40
39
|
use std::time::Instant;
|
41
40
|
use tracing::*;
|
42
41
|
|
42
|
+
struct SearchRequest<'a> {
|
43
|
+
request: &'a VectorSearchRequest,
|
44
|
+
formula: Formula,
|
45
|
+
}
|
46
|
+
|
47
|
+
impl SearchRequest<'_> {
|
48
|
+
fn with_duplicates(&self) -> bool {
|
49
|
+
self.request.with_duplicates
|
50
|
+
}
|
51
|
+
fn get_filter(&self) -> &Formula {
|
52
|
+
&self.formula
|
53
|
+
}
|
54
|
+
fn get_query(&self) -> &[f32] {
|
55
|
+
&self.request.vector
|
56
|
+
}
|
57
|
+
fn no_results(&self) -> usize {
|
58
|
+
self.request.result_per_page as usize
|
59
|
+
}
|
60
|
+
fn min_score(&self) -> f32 {
|
61
|
+
self.request.min_score
|
62
|
+
}
|
63
|
+
}
|
64
|
+
|
43
65
|
#[derive(Clone)]
|
44
66
|
pub struct ScoredParagraph<'a> {
|
45
67
|
score: f32,
|
46
68
|
paragraph: ParagraphRef<'a>,
|
47
69
|
address: ParagraphAddr,
|
48
|
-
|
70
|
+
segment: &'a OpenSegment,
|
49
71
|
}
|
50
72
|
impl Eq for ScoredParagraph<'_> {}
|
51
73
|
impl std::hash::Hash for ScoredParagraph<'_> {
|
@@ -79,9 +101,9 @@ impl PartialEq for ScoredParagraph<'_> {
|
|
79
101
|
}
|
80
102
|
|
81
103
|
impl<'a> ScoredParagraph<'a> {
|
82
|
-
pub fn new(
|
104
|
+
pub fn new(segment: &'a OpenSegment, address: ParagraphAddr, paragraph: ParagraphRef<'a>, score: f32) -> Self {
|
83
105
|
Self {
|
84
|
-
|
106
|
+
segment,
|
85
107
|
paragraph,
|
86
108
|
score,
|
87
109
|
address,
|
@@ -103,7 +125,7 @@ impl<'a> ScoredParagraph<'a> {
|
|
103
125
|
pub fn vectors(&self) -> Vec<&[u8]> {
|
104
126
|
self.paragraph
|
105
127
|
.vectors(&self.address)
|
106
|
-
.map(|va| self.
|
128
|
+
.map(|va| self.segment.get_vector(va).vector())
|
107
129
|
.collect()
|
108
130
|
}
|
109
131
|
}
|
@@ -160,9 +182,9 @@ impl<'a> Fssc<'a> {
|
|
160
182
|
}
|
161
183
|
}
|
162
184
|
|
163
|
-
pub struct
|
185
|
+
pub struct Searcher {
|
164
186
|
config: VectorConfig,
|
165
|
-
|
187
|
+
open_segments: Vec<OpenSegment>,
|
166
188
|
}
|
167
189
|
|
168
190
|
fn segment_matches(expression: &BooleanExpression<String>, labels: &HashSet<String>) -> bool {
|
@@ -180,24 +202,6 @@ fn segment_matches(expression: &BooleanExpression<String>, labels: &HashSet<Stri
|
|
180
202
|
}
|
181
203
|
}
|
182
204
|
|
183
|
-
impl SearchRequest for (&VectorSearchRequest, Formula) {
|
184
|
-
fn with_duplicates(&self) -> bool {
|
185
|
-
self.0.with_duplicates
|
186
|
-
}
|
187
|
-
fn get_filter(&self) -> &Formula {
|
188
|
-
&self.1
|
189
|
-
}
|
190
|
-
fn get_query(&self) -> &[f32] {
|
191
|
-
&self.0.vector
|
192
|
-
}
|
193
|
-
fn no_results(&self) -> usize {
|
194
|
-
self.0.result_per_page as usize
|
195
|
-
}
|
196
|
-
fn min_score(&self) -> f32 {
|
197
|
-
self.0.min_score
|
198
|
-
}
|
199
|
-
}
|
200
|
-
|
201
205
|
impl TryFrom<ScoredParagraph<'_>> for DocumentScored {
|
202
206
|
type Error = String;
|
203
207
|
fn try_from(paragraph: ScoredParagraph) -> Result<Self, Self::Error> {
|
@@ -216,21 +220,21 @@ impl TryFrom<ScoredParagraph<'_>> for DocumentScored {
|
|
216
220
|
}
|
217
221
|
}
|
218
222
|
|
219
|
-
impl
|
220
|
-
pub fn open(open_data_points: Vec<
|
221
|
-
Ok(
|
223
|
+
impl Searcher {
|
224
|
+
pub fn open(open_data_points: Vec<OpenSegment>, config: VectorConfig) -> VectorR<Searcher> {
|
225
|
+
Ok(Searcher {
|
222
226
|
config,
|
223
|
-
open_data_points,
|
227
|
+
open_segments: open_data_points,
|
224
228
|
})
|
225
229
|
}
|
226
230
|
|
227
231
|
pub fn space_usage(&self) -> usize {
|
228
|
-
self.
|
232
|
+
self.open_segments.iter().map(|dp| dp.space_usage()).sum()
|
229
233
|
}
|
230
234
|
|
231
|
-
|
235
|
+
fn _search(
|
232
236
|
&self,
|
233
|
-
request: &
|
237
|
+
request: &SearchRequest,
|
234
238
|
segment_filter: &Option<BooleanExpression<String>>,
|
235
239
|
) -> VectorR<Vec<ScoredParagraph>> {
|
236
240
|
let normalized_query;
|
@@ -257,21 +261,21 @@ impl Reader {
|
|
257
261
|
let min_score = request.min_score();
|
258
262
|
let mut ffsv = Fssc::new(request.no_results(), with_duplicates);
|
259
263
|
|
260
|
-
for
|
264
|
+
for open_segment in &self.open_segments {
|
261
265
|
// Skip this segment if it doesn't match the segment filter
|
262
266
|
if !segment_filter
|
263
267
|
.as_ref()
|
264
|
-
.is_none_or(|f| segment_matches(f,
|
268
|
+
.is_none_or(|f| segment_matches(f, open_segment.tags()))
|
265
269
|
{
|
266
270
|
continue;
|
267
271
|
}
|
268
272
|
let partial_solution =
|
269
|
-
|
273
|
+
open_segment.search(query, filter, with_duplicates, no_results, &self.config, min_score);
|
270
274
|
|
271
275
|
for candidate in partial_solution {
|
272
276
|
let addr = candidate.paragraph();
|
273
|
-
let paragraph =
|
274
|
-
let scored_paragraph = ScoredParagraph::new(
|
277
|
+
let paragraph = open_segment.get_paragraph(addr);
|
278
|
+
let scored_paragraph = ScoredParagraph::new(open_segment, addr, paragraph, candidate.score());
|
275
279
|
ffsv.add(scored_paragraph, candidate.vector());
|
276
280
|
}
|
277
281
|
}
|
@@ -310,9 +314,10 @@ impl Reader {
|
|
310
314
|
|
311
315
|
let v = time.elapsed().as_millis();
|
312
316
|
debug!("{id:?} - Searching: starts at {v} ms");
|
317
|
+
let search_request = &SearchRequest { request, formula };
|
313
318
|
let result = match self.config.vector_cardinality {
|
314
|
-
VectorCardinality::Single => self.
|
315
|
-
VectorCardinality::Multi => self.search_multi_vector(
|
319
|
+
VectorCardinality::Single => self._search(search_request, &request.segment_filtering_formula)?,
|
320
|
+
VectorCardinality::Multi => self.search_multi_vector(search_request)?,
|
316
321
|
};
|
317
322
|
|
318
323
|
let v = time.elapsed().as_millis();
|
@@ -337,13 +342,8 @@ impl Reader {
|
|
337
342
|
})
|
338
343
|
}
|
339
344
|
|
340
|
-
fn
|
341
|
-
let
|
342
|
-
self._search(&search_request, &request.segment_filtering_formula)
|
343
|
-
}
|
344
|
-
|
345
|
-
fn search_multi_vector(&self, request: &VectorSearchRequest, formula: Formula) -> VectorR<Vec<ScoredParagraph>> {
|
346
|
-
let search_vectors = extract_multi_vectors(&request.vector, &self.config.vector_type)?;
|
345
|
+
fn search_multi_vector(&self, search: &SearchRequest) -> VectorR<Vec<ScoredParagraph>> {
|
346
|
+
let search_vectors = extract_multi_vectors(&search.request.vector, &self.config.vector_type)?;
|
347
347
|
let encoded_query = search_vectors
|
348
348
|
.iter()
|
349
349
|
.map(|v| self.config.vector_type.encode(v))
|
@@ -353,18 +353,21 @@ impl Reader {
|
|
353
353
|
let results = search_vectors
|
354
354
|
.into_par_iter()
|
355
355
|
.map(|v| {
|
356
|
-
let mut
|
356
|
+
let mut request = search.request.clone();
|
357
357
|
|
358
|
-
|
358
|
+
request.vector = v;
|
359
359
|
// We are OK with duplicate individual vectors. We always deduplicate by paragraphs anyway (NodeFilter.paragraphs)
|
360
|
-
|
360
|
+
request.with_duplicates = true;
|
361
361
|
// We don't care about min_score in this first pass, we apply min_score on top of maxsim similarity
|
362
|
-
|
362
|
+
request.min_score = f32::MIN;
|
363
363
|
// Request at least a few vectors, since the rerank may offer different results later
|
364
|
-
|
364
|
+
request.result_per_page = search.request.result_per_page.max(10);
|
365
365
|
|
366
|
-
let search_request =
|
367
|
-
|
366
|
+
let search_request = SearchRequest {
|
367
|
+
request: &request,
|
368
|
+
formula: search.formula.clone(),
|
369
|
+
};
|
370
|
+
self._search(&search_request, &search.request.segment_filtering_formula)
|
368
371
|
})
|
369
372
|
.collect::<Result<Vec<_>, _>>()?;
|
370
373
|
|
@@ -379,13 +382,13 @@ impl Reader {
|
|
379
382
|
.into_par_iter()
|
380
383
|
.filter_map(|mut sp| {
|
381
384
|
sp.score = maxsim_similarity(similarity_function, &encoded_query, &sp.vectors());
|
382
|
-
(sp.score() > request.min_score).then_some(sp)
|
385
|
+
(sp.score() > search.request.min_score).then_some(sp)
|
383
386
|
})
|
384
387
|
.collect::<Vec<_>>();
|
385
388
|
|
386
389
|
// Select top_k
|
387
390
|
results.sort_unstable_by(|a, b| b.score().partial_cmp(&a.score()).unwrap());
|
388
|
-
results.truncate(request.result_per_page as usize);
|
391
|
+
results.truncate(search.request.result_per_page as usize);
|
389
392
|
|
390
393
|
Ok(results)
|
391
394
|
}
|
@@ -402,8 +405,8 @@ mod tests {
|
|
402
405
|
|
403
406
|
use super::*;
|
404
407
|
use crate::config::{Similarity, VectorCardinality, VectorConfig, VectorType};
|
405
|
-
use crate::data_point;
|
406
408
|
use crate::indexer::{ResourceWrapper, index_resource};
|
409
|
+
use crate::segment;
|
407
410
|
|
408
411
|
#[test]
|
409
412
|
fn test_key_prefix_search() -> anyhow::Result<()> {
|
@@ -474,7 +477,7 @@ mod tests {
|
|
474
477
|
|
475
478
|
let segment = index_resource(ResourceWrapper::from(&resource), segment_path, &vsc)?.unwrap();
|
476
479
|
|
477
|
-
let
|
480
|
+
let searcher = Searcher::open(vec![segment::open(segment, &vsc)?], vsc).unwrap();
|
478
481
|
let request = VectorSearchRequest {
|
479
482
|
id: "".to_string(),
|
480
483
|
vector_set: "".to_string(),
|
@@ -487,13 +490,13 @@ mod tests {
|
|
487
490
|
resource_id: uuid::Uuid::parse_str("6c5fc1f7a69042d4b24b7f18ea354b4a").unwrap(),
|
488
491
|
field_id: "/f/field1".to_string(),
|
489
492
|
};
|
490
|
-
let result =
|
493
|
+
let result = searcher
|
491
494
|
.search(&request, &PrefilterResult::Some(vec![field_filter.clone()]))
|
492
495
|
.unwrap();
|
493
496
|
assert_eq!(result.documents.len(), 4);
|
494
497
|
|
495
498
|
field_filter.field_id = "/f/field2".to_string();
|
496
|
-
let result =
|
499
|
+
let result = searcher
|
497
500
|
.search(&request, &PrefilterResult::Some(vec![field_filter]))
|
498
501
|
.unwrap();
|
499
502
|
assert_eq!(result.documents.len(), 0);
|
@@ -570,7 +573,7 @@ mod tests {
|
|
570
573
|
|
571
574
|
let segment = index_resource(ResourceWrapper::from(&resource), segment_path, &vsc)?.unwrap();
|
572
575
|
|
573
|
-
let
|
576
|
+
let searcher = Searcher::open(vec![segment::open(segment, &vsc)?], vsc).unwrap();
|
574
577
|
let request = VectorSearchRequest {
|
575
578
|
id: "".to_string(),
|
576
579
|
vector_set: "".to_string(),
|
@@ -579,7 +582,7 @@ mod tests {
|
|
579
582
|
with_duplicates: true,
|
580
583
|
..Default::default()
|
581
584
|
};
|
582
|
-
let result =
|
585
|
+
let result = searcher.search(&request, &PrefilterResult::All).unwrap();
|
583
586
|
assert_eq!(result.documents.len(), 4);
|
584
587
|
|
585
588
|
let request = VectorSearchRequest {
|
@@ -590,7 +593,7 @@ mod tests {
|
|
590
593
|
with_duplicates: false,
|
591
594
|
..Default::default()
|
592
595
|
};
|
593
|
-
let result =
|
596
|
+
let result = searcher.search(&request, &PrefilterResult::All).unwrap();
|
594
597
|
assert_eq!(result.documents.len(), 3);
|
595
598
|
|
596
599
|
// Check that min_score works
|
@@ -603,7 +606,7 @@ mod tests {
|
|
603
606
|
min_score: 900.0,
|
604
607
|
..Default::default()
|
605
608
|
};
|
606
|
-
let result =
|
609
|
+
let result = searcher.search(&request, &PrefilterResult::All).unwrap();
|
607
610
|
assert_eq!(result.documents.len(), 0);
|
608
611
|
|
609
612
|
let bad_request = VectorSearchRequest {
|
@@ -614,7 +617,7 @@ mod tests {
|
|
614
617
|
with_duplicates: false,
|
615
618
|
..Default::default()
|
616
619
|
};
|
617
|
-
assert!(
|
620
|
+
assert!(searcher.search(&bad_request, &PrefilterResult::All).is_err());
|
618
621
|
|
619
622
|
Ok(())
|
620
623
|
}
|
@@ -680,7 +683,7 @@ mod tests {
|
|
680
683
|
|
681
684
|
let segment = index_resource(ResourceWrapper::from(&resource), segment_path, &vsc)?.unwrap();
|
682
685
|
|
683
|
-
let
|
686
|
+
let searcher = Searcher::open(vec![segment::open(segment, &vsc)?], vsc).unwrap();
|
684
687
|
let request = VectorSearchRequest {
|
685
688
|
id: "".to_string(),
|
686
689
|
vector_set: "".to_string(),
|
@@ -689,7 +692,7 @@ mod tests {
|
|
689
692
|
with_duplicates: true,
|
690
693
|
..Default::default()
|
691
694
|
};
|
692
|
-
let result =
|
695
|
+
let result = searcher.search(&request, &PrefilterResult::All).unwrap();
|
693
696
|
assert_eq!(result.documents.len(), 2);
|
694
697
|
|
695
698
|
let request = VectorSearchRequest {
|
@@ -700,7 +703,7 @@ mod tests {
|
|
700
703
|
with_duplicates: false,
|
701
704
|
..Default::default()
|
702
705
|
};
|
703
|
-
let result =
|
706
|
+
let result = searcher.search(&request, &PrefilterResult::All).unwrap();
|
704
707
|
assert_eq!(result.documents.len(), 1);
|
705
708
|
|
706
709
|
Ok(())
|