nidx-binding 6.3.5.post274__tar.gz → 6.3.5.post281__tar.gz

{nidx_binding-6.3.5.post274 → nidx_binding-6.3.5.post281}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nidx_binding
-Version: 6.3.5.post274
+Version: 6.3.5.post281
 Classifier: Programming Language :: Rust
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy

{nidx_binding-6.3.5.post274 → nidx_binding-6.3.5.post281}/nidx_protos/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "pdm.backend"
 
 [project]
 name = "nidx_protos"
-version = "6.3.5.post274"
+version = "6.3.5.post281"
 license = { text = "AGPL" }
 description = "Protobuf definitions for nucliadb/nidx"
 authors = [ { name = "Nuclia", email = "nucliadb@nuclia.com" }]

{nidx_binding-6.3.5.post274 → nidx_binding-6.3.5.post281}/pyproject.toml

@@ -13,7 +13,7 @@ classifiers = [
     "Programming Language :: Python :: Implementation :: CPython",
     "Programming Language :: Python :: Implementation :: PyPy",
 ]
-version = "6.3.5.post274"
+version = "6.3.5.post281"
 
 [project.urls]
 Homepage = "https://nuclia.com"

{nidx_binding-6.3.5.post274 → nidx_binding-6.3.5.post281}/src/indexer.rs

@@ -264,7 +264,7 @@ pub async fn download_message(storage: Arc<DynObjectStore>, storage_key: &str) -
     Ok(resource)
 }
 
-type IndexingResult = (Segment, usize, Vec<String>);
+type IndexingResult = (IndexId, Option<(Segment, usize)>, Vec<String>);
 
 #[instrument(skip_all, fields(shard_id = shard_id))]
 pub async fn index_resource(
@@ -285,7 +285,7 @@ pub async fn index_resource(
     let num_vector_indexes = indexes.iter().filter(|i| matches!(i.kind, IndexKind::Vector)).count();
     let single_vector_index = num_vector_indexes == 1;
 
-    let mut tasks: JoinSet<anyhow::Result<Option<IndexingResult>>> = JoinSet::new();
+    let mut tasks: JoinSet<anyhow::Result<IndexingResult>> = JoinSet::new();
     for index in indexes {
         let resource = Arc::clone(&resource);
         let meta = meta.clone();
@@ -302,7 +302,7 @@ pub async fn index_resource(
                 })
                 .await??;
                 let Some(new_segment) = new_segment else {
-                    return Ok(None);
+                    return Ok((index.id, None, deletions));
                 };
 
                 // Create the segment first so we can track it if the upload gets interrupted
@@ -316,7 +316,7 @@ pub async fn index_resource(
                 .await?;
                 let size = pack_and_upload(storage.clone(), output_dir.path(), segment.id.storage_key()).await?;
 
-                Ok(Some((segment, size, deletions)))
+                Ok((index.id, Some((segment, size)), deletions))
             }
             .in_current_span(),
         );
@@ -328,13 +328,19 @@ pub async fn index_resource(
     // and the index message can be safely reprocessed
     let mut tx = meta.transaction().await?;
     let mut indexes = Vec::new();
-    for (segment, size, deletions) in results?.into_iter().flatten() {
-        // Mark the segments as visible and write the deletions at the same time
-        segment.mark_ready(&mut *tx, size as i64).await?;
+    for (index_id, segment_result, deletions) in results?.into_iter() {
+        let mut update_index = false;
+        if let Some((segment, size)) = segment_result {
+            segment.mark_ready(&mut *tx, size as i64).await?;
+            update_index = true;
+        }
         if !deletions.is_empty() {
-            Deletion::create(&mut *tx, segment.index_id, seq, &deletions).await?;
+            Deletion::create(&mut *tx, index_id, seq, &deletions).await?;
+            update_index = true;
+        }
+        if update_index {
+            indexes.push(index_id)
         }
-        indexes.push(segment.index_id);
     }
     Index::updated_many(&mut *tx, &indexes).await?;
     tx.commit().await?;
@@ -486,4 +492,41 @@ mod tests {
 
         Ok(())
     }
+
+    #[sqlx::test]
+    async fn test_index_deletions_empty_segment(pool: sqlx::PgPool) -> anyhow::Result<()> {
+        let meta = NidxMetadata::new_with_pool(pool).await?;
+        let storage = Arc::new(object_store::memory::InMemory::new());
+
+        let kbid = Uuid::new_v4();
+        let shard = Shard::create(&meta.pool, kbid).await?;
+        let index = Index::create(&meta.pool, shard.id, "multilingual", VECTOR_CONFIG.into()).await?;
+
+        // Index a resource with only deletions -- no new segment is created
+        let mut resource = minimal_resource(shard.id.to_string());
+        resource.texts_to_delete.push("uuid/t/title".to_string());
+        resource.vector_prefixes_to_delete.insert(
+            "multilingual".to_string(),
+            StringList {
+                items: vec!["uuid/t/title".to_string()],
+            },
+        );
+        resource.paragraphs_to_delete.push("uuid/t/title".to_string());
+        resource.relation_fields_to_delete.push("uuid/t/title".to_string());
+
+        index_resource(
+            &meta,
+            storage.clone(),
+            &tempfile::env::temp_dir(),
+            &shard.id.to_string(),
+            resource.clone(),
+            1i64.into(),
+        )
+        .await?;
+
+        let deletions = Deletion::for_index_and_seq(&meta.pool, index.id, 10i64.into()).await?;
+        assert!(!deletions.is_empty());
+
+        Ok(())
+    }
 }