nucliadb 6.2.1.post3025__py3-none-any.whl → 6.2.1.post3034__py3-none-any.whl

nucliadb/ingest/fields/base.py

@@ -43,6 +43,7 @@ from nucliadb_protos.resources_pb2 import (
 )
 from nucliadb_protos.utils_pb2 import ExtractedText, VectorObject
 from nucliadb_protos.writer_pb2 import Error, FieldStatus
+from nucliadb_utils.storages.exceptions import CouldNotCopyNotFound
 from nucliadb_utils.storages.storage import Storage, StorageField
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -370,9 +371,26 @@ class Field(Generic[PbType]):
         sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
         vo: Optional[VectorObject] = None
         if actual_payload is None:
-            # Its first extracted text
+            # Its first extracted vectors
             if payload.HasField("file"):
-                await self.storage.normalize_binary(payload.file, sf)
+                # When we receive vectors in a cloud file, it points to our
+                # storage but paths are different, we may want to move it. This
+                # can happen, for example, with LEGACY KBs where processing
+                # sends us the extracted vectors prefixed by vectorset but, to
+                # maintain bw/c, we move those to the original not prefixed
+                # path.
+                try:
+                    await self.storage.normalize_binary(payload.file, sf)
+                except CouldNotCopyNotFound:
+                    # A failure here could mean the payload has already been
+                    # moved and we're retrying due to a redelivery or another
+                    # retry mechanism
+                    already_moved = await sf.exists()
+                    if already_moved:
+                        # We assume is the correct one and do nothing else
+                        pass
+                    else:
+                        raise
                 vo = await self.storage.download_pb(sf, VectorObject)
             else:
                 await self.storage.upload_pb(sf, payload.vectors)

nucliadb/ingest/orm/knowledgebox.py

@@ -59,13 +59,11 @@ from nucliadb_protos.knowledgebox_pb2 import (
     VectorSetPurge,
 )
 from nucliadb_protos.resources_pb2 import Basic
-from nucliadb_utils import const
 from nucliadb_utils.settings import is_onprem_nucliadb
 from nucliadb_utils.storages.storage import Storage
 from nucliadb_utils.utilities import (
     get_audit,
     get_storage,
-    has_feature,
 )
 
 # XXX Eventually all these keys should be moved to datamanagers.kb
@@ -149,31 +147,6 @@ class KnowledgeBox:
 
                 vs_external_indexes = []
 
-                # HACK! Currently, we share responsibility of deciding where to
-                # store extracted vectors with processing. Depending on whether
-                # it sends the vectorset id or not (in the extracted vectors
-                # wrapper) nucliadb will store the extracted vectors in a different place
-                #
-                # Right now, processing behaviour is not setting vectorset ids
-                # if there's only one semantic model configured. This is done
-                # for Bw/c with KBs previous to vectorsets.
-                #
-                # We now hardcode this assumption here, so we can annotate each
-                # vectorset with a mark and don't depend on processing
-                # information to decide the storage key. Once this is done we'll
-                # be able to force processing to always send vectorset ids and
-                # remove that bw/c behavior
-                #
-                if has_feature(const.Features.REMOVE_DEFAULT_VECTORSET):
-                    storage_key_kind = knowledgebox_pb2.VectorSetConfig.StorageKeyKind.VECTORSET_PREFIX
-                else:
-                    if len(semantic_models) == 1:
-                        storage_key_kind = knowledgebox_pb2.VectorSetConfig.StorageKeyKind.LEGACY
-                    else:
-                        storage_key_kind = (
-                            knowledgebox_pb2.VectorSetConfig.StorageKeyKind.VECTORSET_PREFIX
-                        )
-
                 for vectorset_id, semantic_model in semantic_models.items():  # type: ignore
                     # if this KB uses a matryoshka model, we can choose a different
                     # dimension
@@ -200,7 +173,7 @@ class KnowledgeBox:
                             vector_dimension=dimension,
                         ),
                         matryoshka_dimensions=semantic_model.matryoshka_dimensions,
-                        storage_key_kind=storage_key_kind,
+                        storage_key_kind=knowledgebox_pb2.VectorSetConfig.StorageKeyKind.VECTORSET_PREFIX,
                     )
                     await datamanagers.vectorsets.set(txn, kbid=kbid, config=vectorset_config)
 

{nucliadb-6.2.1.post3025.dist-info → nucliadb-6.2.1.post3034.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post3025
+Version: 6.2.1.post3034
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3025
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3025
-Requires-Dist: nucliadb-protos>=6.2.1.post3025
-Requires-Dist: nucliadb-models>=6.2.1.post3025
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3034
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3034
+Requires-Dist: nucliadb-protos>=6.2.1.post3034
+Requires-Dist: nucliadb-models>=6.2.1.post3034
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.2.1.post3025.dist-info → nucliadb-6.2.1.post3034.dist-info}/RECORD

@@ -118,7 +118,7 @@ nucliadb/ingest/consumer/service.py,sha256=EZM1sABW_7bj6j2UgKUHUuK-EGIEYnLdtPAn8
 nucliadb/ingest/consumer/shard_creator.py,sha256=8SotMc-o_G8XZU52gR4Aay7tcigTdIXgz8YtxqHmJ1Q,4309
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
 nucliadb/ingest/fields/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
-nucliadb/ingest/fields/base.py,sha256=b6QpVPsCiDirDiYG3-yOCMaSNznJSHmQB0z6J_eDIyw,20657
+nucliadb/ingest/fields/base.py,sha256=DTXFu_g9vSU2h7S4PlYxd-hp4SDuM_EPf8y51ALgD8w,21654
 nucliadb/ingest/fields/conversation.py,sha256=OcQOHvi72Pm0OyNGwxLo9gONo8f1NhwASq0_gS-E64A,7021
 nucliadb/ingest/fields/exceptions.py,sha256=LBZ-lw11f42Pk-ck-NSN9mSJ2kOw-NeRwb-UE31ILTQ,1171
 nucliadb/ingest/fields/file.py,sha256=1v4jLg3balUua2VmSV8hHkAwPFShTUCOzufZvIUQcQw,4740
@@ -130,7 +130,7 @@ nucliadb/ingest/orm/brain.py,sha256=UND5EsNUdd7XdjScYqRqg4r_xCx3l-My8alGw5M9CWg,
 nucliadb/ingest/orm/broker_message.py,sha256=ZEMueoGuuRKO4tHgzc0P0AM1Ls1TTYey_4UvRQf0BpY,6915
 nucliadb/ingest/orm/entities.py,sha256=5d6Gfo-Yz-rns_mNJeRqiGaPeWpUMgSKZnmWIGMLCKo,15537
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/knowledgebox.py,sha256=CGNfvft-IzSUQNENsnqYebp5dGfk2o3N561pcKes-28,25144
+nucliadb/ingest/orm/knowledgebox.py,sha256=IGOPvBR1qXqDxE5DeiOdYCLdPgjzOVVpsASJ2zYvWwQ,23651
 nucliadb/ingest/orm/metrics.py,sha256=OkwMSPKLZcKba0ZTwtTiIxwBgaLMX5ydhGieKvi2y7E,1096
 nucliadb/ingest/orm/resource.py,sha256=KDTEwZ6_5eLvLu1s30Pln4BFDOy9D7_ChT7kRmQ1J2g,44670
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
@@ -330,9 +330,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post3025.dist-info/METADATA,sha256=9HhgfxRf98PfKRLxtV8WwdwVuN1X-P-L2wkYg_ZNJsw,4603
-nucliadb-6.2.1.post3025.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post3025.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post3025.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post3025.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post3025.dist-info/RECORD,,
+nucliadb-6.2.1.post3034.dist-info/METADATA,sha256=NQ9r_0EVqvRG-HjOJ549zBC_gi16eZxAz8FpZTpmCk8,4603
+nucliadb-6.2.1.post3034.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post3034.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post3034.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post3034.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post3034.dist-info/RECORD,,