nidx-binding 6.3.1.post17__tar.gz → 6.3.1.post20__tar.gz

{nidx_binding-6.3.1.post17 → nidx_binding-6.3.1.post20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.1.post17
+Version: 6.3.1.post20
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.1.post17 → nidx_binding-6.3.1.post20}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.1.post17"
+version = "6.3.1.post20"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.1.post17 → nidx_binding-6.3.1.post20}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.1.post17"
+version = "6.3.1.post20"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.1.post17 → nidx_binding-6.3.1.post20}/tests/common/services.rs

@@ -19,15 +19,14 @@
 //
 
 use std::sync::Arc;
-use std::time::Duration;
 
 use nidx::api::grpc::ApiServer;
 use nidx::grpc_server::GrpcServer;
 use nidx::indexer::index_resource;
 use nidx::searcher::grpc::SearchServer;
 use nidx::searcher::shard_selector::ShardSelector;
-use nidx::searcher::SyncedSearcher;
-use nidx::settings::{EnvSettings, MetadataSettings, StorageSettings};
+use nidx::searcher::{SyncStatus, SyncedSearcher};
+use nidx::settings::{EnvSettings, MetadataSettings, SearcherSettings, StorageSettings};
 use nidx::{NidxMetadata, Settings};
 use nidx_protos::nidx::nidx_api_client::NidxApiClient;
 use nidx_protos::nidx::nidx_searcher_client::NidxSearcherClient;
@@ -35,6 +34,7 @@ use nidx_protos::Resource;
 use object_store::memory::InMemory;
 use sqlx::PgPool;
 use tempfile::tempdir;
+use tokio::sync::watch;
 use tokio_util::sync::CancellationToken;
 use tonic::transport::Channel;
 
@@ -43,6 +43,7 @@ pub struct NidxFixture {
     pub api_client: NidxApiClient<Channel>,
     pub settings: Settings,
     seq: i64,
+    sync_watcher: watch::Receiver<SyncStatus>,
 }
 
 impl NidxFixture {
@@ -65,7 +66,11 @@ impl NidxFixture {
                 telemetry: Default::default(),
                 work_path: None,
                 control_socket: None,
-                searcher: Default::default(),
+                searcher: Some(SearcherSettings {
+                    // Reduced refresh interval for faster testing
+                    metadata_refresh_interval: 0.1,
+                    ..Default::default()
+                }),
                 audit: None,
             },
         };
@@ -83,6 +88,7 @@ impl NidxFixture {
         let searcher_port = searcher_server.port()?;
         tokio::task::spawn(searcher_server.serve(searcher_api.into_router(), shutdown.clone()));
         let settings_copy = settings.clone();
+        let (sync_reporter, sync_watcher) = watch::channel(SyncStatus::Syncing);
         tokio::task::spawn(async move {
             searcher
                 .run(
@@ -90,7 +96,7 @@ impl NidxFixture {
                     settings_copy.searcher.clone().unwrap_or_default(),
                     shutdown.clone(),
                     ShardSelector::new_single(),
-                    None,
+                    Some(sync_reporter),
                     None,
                 )
                 .await
@@ -105,6 +111,7 @@ impl NidxFixture {
             api_client,
             settings,
             seq: 1,
+            sync_watcher,
         })
     }
 
@@ -122,8 +129,10 @@ impl NidxFixture {
         Ok(())
     }
 
-    pub async fn wait_sync(&self) {
-        // TODO: Check the searcher has synced? For now, waiting twice the sync interval
-        tokio::time::sleep(Duration::from_secs(2)).await;
+    pub async fn wait_sync(&mut self) {
+        // Wait for a new sync to start
+        self.sync_watcher.wait_for(|s| matches!(s, SyncStatus::Syncing)).await.unwrap();
+        // Wait for it to finish
+        self.sync_watcher.wait_for(|s| matches!(s, SyncStatus::Synced)).await.unwrap();
     }
 }

{nidx_binding-6.3.1.post17 → nidx_binding-6.3.1.post20}/tests/test_searcher_cluster.rs

@@ -97,7 +97,7 @@ async fn test_search_cluster_all_shards_accessible(pool: PgPool) -> anyhow::Resu
                 .await
         });
     }
-    tokio::time::sleep(Duration::from_secs(1)).await;
+    tokio::time::sleep(Duration::from_millis(150)).await;
 
     // All shards are accessible from all nodes
     for shard in &shards {
@@ -117,7 +117,7 @@ async fn test_search_cluster_all_shards_accessible(pool: PgPool) -> anyhow::Resu
 
     // Remove one node from the cluster and wait for sync, searches should still work
     nodes.lock().unwrap().remove(2);
-    tokio::time::sleep(Duration::from_secs_f32(1.2)).await;
+    tokio::time::sleep(Duration::from_millis(150)).await;
     for shard in &shards {
         for searcher in &searchers[0..2] {
             NidxSearcherClient::connect(format!("http://{searcher}"))
@@ -187,7 +187,7 @@ async fn test_search_cluster_shard_distribution(pool: PgPool) -> anyhow::Result<
                 .await
         });
     }
-    tokio::time::sleep(Duration::from_secs(1)).await;
+    tokio::time::sleep(Duration::from_millis(150)).await;
 
     // Each shard is only accessible from a single node
     let mut node_for_shard_original = HashMap::new();
@@ -219,7 +219,7 @@ async fn test_search_cluster_shard_distribution(pool: PgPool) -> anyhow::Result<
 
     // Remove one node from the cluster, we expect shards to rebalance among the remaining nodes
     nodes.lock().unwrap().remove(2);
-    tokio::time::sleep(Duration::from_secs_f32(1.2)).await;
+    tokio::time::sleep(Duration::from_millis(150)).await;
 
     let mut node_for_shard_scale_down = HashMap::new();
     for shard in &shards {
@@ -257,7 +257,7 @@ async fn test_search_cluster_shard_distribution(pool: PgPool) -> anyhow::Result<
     // This node will have a different ID, so shard split
     // should be different than originally
     nodes.lock().unwrap().push("fake_new_node".to_string());
-    tokio::time::sleep(Duration::from_secs_f32(1.2)).await;
+    tokio::time::sleep(Duration::from_millis(150)).await;
 
     let mut node_for_shard_scale_up = HashMap::new();
     for shard in &shards {

{nidx_binding-6.3.1.post17 → nidx_binding-6.3.1.post20}/tests/test_synced_searcher.rs

@@ -54,16 +54,13 @@ async fn test_synchronization(pool: sqlx::PgPool) -> anyhow::Result<()> {
     let synced_searcher = SyncedSearcher::new(meta.clone(), work_dir.path());
     let index_cache = synced_searcher.index_cache();
     let storage_copy = storage.clone();
+    let settings = SearcherSettings {
+        metadata_refresh_interval: 0.1,
+        ..Default::default()
+    };
     let search_task = tokio::spawn(async move {
         synced_searcher
-            .run(
-                storage_copy,
-                SearcherSettings::default(),
-                CancellationToken::new(),
-                ShardSelector::new_single(),
-                None,
-                None,
-            )
+            .run(storage_copy, settings, CancellationToken::new(), ShardSelector::new_single(), None, None)
             .await
     });
 
@@ -96,7 +93,7 @@ async fn test_synchronization(pool: sqlx::PgPool) -> anyhow::Result<()> {
     )
     .await?;
 
-    tokio::time::sleep(Duration::from_secs(2)).await;
+    tokio::time::sleep(Duration::from_millis(200)).await;
 
     // TODO: Test with shard_search once we have more indexes. We can make IndexSearch enum private again
     let searcher = index_cache.get(&index.id).await?;
@@ -115,7 +112,7 @@ async fn test_synchronization(pool: sqlx::PgPool) -> anyhow::Result<()> {
     process_index_message(&meta, storage.clone(), storage.clone(), &tempfile::env::temp_dir(), deletion, 2i64.into())
         .await?;
 
-    tokio::time::sleep(Duration::from_secs(2)).await;
+    tokio::time::sleep(Duration::from_millis(200)).await;
 
     let searcher = index_cache.get(&index.id).await?;
     let vector_searcher: &VectorSearcher = searcher.as_ref().into();