nucliadb 6.2.1.post2751__py3-none-any.whl → 6.2.1.post2755__py3-none-any.whl

nucliadb/common/external_index_providers/pinecone.py

@@ -441,6 +441,7 @@ class PineconeIndexManager(ExternalIndexManager):
 
     def get_prefixes_to_delete(self, index_data: Resource) -> set[str]:
         prefixes_to_delete = set()
+        # TODO: migrate to vector_prefixes_to_delete
         for field_id in index_data.sentences_to_delete:
             try:
                 delete_vid = VectorId.from_string(field_id)

nucliadb/ingest/orm/brain.py

@@ -100,6 +100,8 @@ class ResourceBrain:
         page_positions: Optional[FilePagePositions],
         extracted_text: Optional[ExtractedText],
         basic_user_field_metadata: Optional[UserFieldMetadata] = None,
+        *,
+        replace_field: bool = False,
     ):
         # To check for duplicate paragraphs
         unique_paragraphs: set[str] = set()
@@ -224,6 +226,11 @@ class ResourceBrain:
 
             self.brain.paragraphs[field_key].paragraphs[key].CopyFrom(p)
 
+        if replace_field:
+            field_type, field_name = field_key.split("/")
+            full_field_id = ids.FieldId(rid=self.rid, type=field_type, key=field_name).full()
+            self.brain.paragraphs_to_delete.append(full_field_id)
+
         for relations in metadata.metadata.relations:
             for relation in relations.relations:
                 self.brain.relations.append(relation)
@@ -301,8 +308,11 @@ class ResourceBrain:
 
         if replace_field:
             full_field_id = ids.FieldId(rid=self.rid, type=fid.type, key=fid.key).full()
-            self.brain.sentences_to_delete.append(full_field_id)
-            self.brain.paragraphs_to_delete.append(full_field_id)
+            if vectorset is None:
+                # DEPRECATED
+                self.brain.sentences_to_delete.append(full_field_id)
+            else:
+                self.brain.vector_prefixes_to_delete[vectorset].items.append(full_field_id)
 
     def _apply_field_vector(
         self,

nucliadb/ingest/orm/processor/processor.py

@@ -275,7 +275,6 @@ class Processor:
 
                     if message.source == writer_pb2.BrokerMessage.MessageSource.WRITER:
                         resource = await kb.get(uuid)
-
                         if resource is None:
                             # It's a new resource
                             resource = await kb.add_resource(uuid, message.slug, message.basic)
@@ -737,7 +736,11 @@ def has_vectors_operation(index_message: PBBrainResource) -> bool:
     """
     Returns True if the index message has any vectors to index or to delete.
     """
-    if len(index_message.sentences_to_delete) > 0 or len(index_message.paragraphs_to_delete) > 0:
+    if (
+        len(index_message.sentences_to_delete) > 0
+        or len(index_message.paragraphs_to_delete) > 0
+        or any([len(deletions.items) for deletions in index_message.vector_prefixes_to_delete.values()])
+    ):
         return True
     for field_paragraphs in index_message.paragraphs.values():
         for paragraph in field_paragraphs.paragraphs.values():

nucliadb/ingest/orm/resource.py

@@ -226,6 +226,7 @@ class Resource:
                             page_positions=page_positions,
                             extracted_text=await field_obj.get_extracted_text(),
                             basic_user_field_metadata=user_field_metadata,
+                            replace_field=True,
                         )
 
         # Some basic fields are computed off field metadata.
@@ -336,6 +337,7 @@ class Resource:
                     page_positions=page_positions,
                     extracted_text=await field.get_extracted_text(),
                     basic_user_field_metadata=user_field_metadata,
+                    replace_field=reindex,
                 )
 
             if self.disable_vectors is False:
@@ -584,6 +586,7 @@ class Resource:
         # Upload to binary storage
         # Vector indexing
         if self.disable_vectors is False:
+            await self.get_fields(force=True)
             for field_vectors in message.field_vectors:
                 await self._apply_extracted_vectors(field_vectors)
 
@@ -723,6 +726,7 @@ class Resource:
             page_positions=page_positions,
             extracted_text=extracted_text,
             basic_user_field_metadata=user_field_metadata,
+            replace_field=True,
         )
         loop = asyncio.get_running_loop()
         await loop.run_in_executor(_executor, apply_field_metadata)

nucliadb/writer/api/v1/knowledgebox.py

@@ -68,6 +68,7 @@ async def create_kb_endpoint(request: Request, item: KnowledgeBoxConfig) -> Know
     except ExternalIndexCreationError as exc:
         raise HTTPException(status_code=502, detail=str(exc))
     except Exception:
+        logger.exception("Could not create KB")
         raise HTTPException(status_code=500, detail="Error creating knowledge box")
     else:
         return KnowledgeBoxObj(uuid=kbid, slug=slug)

{nucliadb-6.2.1.post2751.dist-info → nucliadb-6.2.1.post2755.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post2751
+Version: 6.2.1.post2755
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2751
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2751
-Requires-Dist: nucliadb-protos>=6.2.1.post2751
-Requires-Dist: nucliadb-models>=6.2.1.post2751
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2755
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2755
+Requires-Dist: nucliadb-protos>=6.2.1.post2755
+Requires-Dist: nucliadb-models>=6.2.1.post2755
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: nuclia-models>=0.24.2

{nucliadb-6.2.1.post2751.dist-info → nucliadb-6.2.1.post2755.dist-info}/RECORD

@@ -82,7 +82,7 @@ nucliadb/common/external_index_providers/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A
 nucliadb/common/external_index_providers/base.py,sha256=jijdPxWNLUUO3xZISpO7vvlmm8rOxNY56z8s6QVys6o,8809
 nucliadb/common/external_index_providers/exceptions.py,sha256=nDhhOIkb66hjCrBk4Spvl2vN1SuW5gbwrMCDmrdjHHE,1209
 nucliadb/common/external_index_providers/manager.py,sha256=aFSrrKKYG1ydpTSyq4zYD0LOxFS7P-CO6rcKC0hiF4I,4267
-nucliadb/common/external_index_providers/pinecone.py,sha256=Ux5P4JZ7eVOr1fl1ih7qxbXr_Up_5G7FzUhhJezpaVk,39749
+nucliadb/common/external_index_providers/pinecone.py,sha256=afglJq6FfifFNTONGrFBCe5yuEL2h3pDFwZkKiA4_6o,39802
 nucliadb/common/external_index_providers/settings.py,sha256=EGHnIkwxqe6aypwKegXTlKO3AgUxNa-6GeAZG25Njis,2002
 nucliadb/common/http_clients/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/common/http_clients/auth.py,sha256=srfpgAbs2wmqA9u_l-HxsV4YoO77Tse4y3gm3q2YvYM,2112
@@ -134,19 +134,19 @@ nucliadb/ingest/fields/generic.py,sha256=elgtqv15aJUq3zY7X_g0bli_2BpcwPArVvzhe54
 nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJs,4172
 nucliadb/ingest/fields/text.py,sha256=tFvSQJAe0W7ePpp2_WDfLiE2yglR1OTU0Zht9acvOFw,1594
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/orm/brain.py,sha256=5n1m2ysTC5dlzOUL2cBa27wCLG9wDqfe85wK7id80AI,28437
+nucliadb/ingest/orm/brain.py,sha256=Hzq-3aarKaUCiUoa8H83unRUfduRE9TsQH1dEq0mvZY,28841
 nucliadb/ingest/orm/broker_message.py,sha256=JYYUJIZEL_EqovQuw6u-FmEkjyoYlxIXJq9hFekOiks,6441
 nucliadb/ingest/orm/entities.py,sha256=2PslT1FZ6yCvJtjR0UpKTSzxJrtS-C_gZx4ZTWHunTc,15759
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
 nucliadb/ingest/orm/knowledgebox.py,sha256=dBetjoJBYT6JuGmMHiqjcfJeD8qJrK3MQt9X03IrHRA,23228
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
-nucliadb/ingest/orm/resource.py,sha256=_m4B14dSpO-lszpoqlhXYL3LrplB9p3NrDZC5kQbXHs,53860
+nucliadb/ingest/orm/resource.py,sha256=ZiCWa-ayznr3XRnYN7v7_RoYpwvoKddky5N4QRBTjhY,54029
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=ZF66gfHHMHCh9n9XXoTSdTXhETnzJlcylfQqhFq54Pw,5775
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=f32PIEXWktWzGDws6Ffife37OAfrseP5IOti_Cb4ir8,3012
-nucliadb/ingest/orm/processor/processor.py,sha256=Wmh_7KUT2U0oX1pMyzw1jVHNMVGoaBn-6cDYqNoTHtA,30817
+nucliadb/ingest/orm/processor/processor.py,sha256=2FxAetUvtHvg6l-24xYrmBdsyqc0RU1zmliel44581g,30945
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=MME_G_ERxzJR6JW_hfE2qcfXpmpH1kdG-S0a-M0qRm8,2043
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -313,7 +313,7 @@ nucliadb/writer/api/utils.py,sha256=wIQHlU8RQiIGVLI72suvyVIKlCU44Unh0Ae0IiN6Qwo,
 nucliadb/writer/api/v1/__init__.py,sha256=FVn7N9VJ6bsEoy4TRnkclr4Umd5hECiwPXVqRnJ8BME,1095
 nucliadb/writer/api/v1/export_import.py,sha256=6_gn0-emCjmK6bCUX5kgMvG0qkZr4HlfGmBXhhngsxo,8243
 nucliadb/writer/api/v1/field.py,sha256=PT5NfnD6TmX0oh-ehGiMJ7YiEtduIrMalqysQANjciY,17213
-nucliadb/writer/api/v1/knowledgebox.py,sha256=WjB-rqrx0SKob6MJSxohSXyhe0V9jr64LrYkJogMBIw,10881
+nucliadb/writer/api/v1/knowledgebox.py,sha256=Mr1vJSWOtiraDdtoTqQ1V2rSirMdojL4wN0Q3cOiX4k,10929
 nucliadb/writer/api/v1/learning_config.py,sha256=GaYaagjBrVG9ZxrWQyVQfqGMQV3tAJjqJ5CStaKhktU,2058
 nucliadb/writer/api/v1/resource.py,sha256=clUu0SZryLIZqvM9URM3fcDNnh6mhRoO7dt4SlRVE8I,18313
 nucliadb/writer/api/v1/router.py,sha256=RjuoWLpZer6Kl2BW_wznpNo6XL3BOpdTGqXZCn3QrrQ,1034
@@ -335,9 +335,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post2751.dist-info/METADATA,sha256=T30boeqSs3RbU0zMbNDAuxTTZsMMXox1Lote5G3Uw3o,4689
-nucliadb-6.2.1.post2751.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post2751.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post2751.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post2751.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post2751.dist-info/RECORD,,
+nucliadb-6.2.1.post2755.dist-info/METADATA,sha256=v8quLHM4YNOOMHBTcxio0COtXxx-fpjNfLhiMRLRQFU,4689
+nucliadb-6.2.1.post2755.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post2755.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post2755.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post2755.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post2755.dist-info/RECORD,,