nucliadb 6.3.4.post3740__py3-none-any.whl → 6.3.4.post3762__py3-none-any.whl

nucliadb/ingest/orm/processor/processor.py

@@ -314,7 +314,7 @@ class Processor:
                 # index message
 
                 if resource:
-                    if message.reindex:
+                    if any(needs_reindex(m) for m in messages):
                         # when reindexing, let's just generate full new index message
                         # TODO - This should be improved in the future as it's not optimal for very large resources:
                         # As of now, there are some API operations that require fully reindexing all the fields of a resource.
@@ -759,3 +759,26 @@ def has_vectors_operation(index_message: PBBrainResource) -> bool:
                 if len(vectorset_sentences.sentences) > 0:
                     return True
     return False
+
+
+def needs_reindex(bm: writer_pb2.BrokerMessage) -> bool:
+    return bm.reindex or is_vectorset_migration_bm(bm)
+
+
+def is_vectorset_migration_bm(bm: writer_pb2.BrokerMessage) -> bool:
+    """
+    This is a temporary solution to avoid duplicating paragraphs and text fields during vector migrations.
+    We need to reindex all the fields of a resource to avoid this issue.
+    TODO: Remove this when the index message generation logic has been decoupled into its own method.
+
+    Broker messages from semantic model migration task only contain the `field_vectors` field set.
+    """
+    return (
+        len(bm.field_vectors) > 0
+        and not bm.HasField("basic")
+        and len(bm.delete_fields) == 0
+        and len(bm.files) == 0
+        and len(bm.texts) == 0
+        and len(bm.conversations) == 0
+        and len(bm.links) == 0
+    )

{nucliadb-6.3.4.post3740.dist-info → nucliadb-6.3.4.post3762.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.3.4.post3740
+Version: 6.3.4.post3762
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.4.post3740
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.4.post3740
-Requires-Dist: nucliadb-protos>=6.3.4.post3740
-Requires-Dist: nucliadb-models>=6.3.4.post3740
-Requires-Dist: nidx-protos>=6.3.4.post3740
+Requires-Dist: nucliadb-telemetry[all]>=6.3.4.post3762
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.4.post3762
+Requires-Dist: nucliadb-protos>=6.3.4.post3762
+Requires-Dist: nucliadb-models>=6.3.4.post3762
+Requires-Dist: nidx-protos>=6.3.4.post3762
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn

{nucliadb-6.3.4.post3740.dist-info → nucliadb-6.3.4.post3762.dist-info}/RECORD

@@ -151,7 +151,7 @@ nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnq
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=HpSU9olDHcTfECDYCsmm4yA-Hu0mBrd_zTtx50XDGFE,5164
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=f32PIEXWktWzGDws6Ffife37OAfrseP5IOti_Cb4ir8,3012
-nucliadb/ingest/orm/processor/processor.py,sha256=wjZmfJumypdGoKVB0BEJ51ha3xNRs1CzDZbFGj7z4_8,31668
+nucliadb/ingest/orm/processor/processor.py,sha256=Tb87wm9Yqd4sS_x_wFTaIG3Tn3sznG8AuLCOanYhVDQ,32536
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=MME_G_ERxzJR6JW_hfE2qcfXpmpH1kdG-S0a-M0qRm8,2043
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -354,8 +354,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.4.post3740.dist-info/METADATA,sha256=kXv8CG9jprFzMIeB6zi4A3R9gUN7RQ9J2CiTGBolbu0,4291
-nucliadb-6.3.4.post3740.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
-nucliadb-6.3.4.post3740.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.4.post3740.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.4.post3740.dist-info/RECORD,,
+nucliadb-6.3.4.post3762.dist-info/METADATA,sha256=Zr2XIrUsS85ApCvFMDHlLy5RwSd_sNZjds-dEClJkqg,4291
+nucliadb-6.3.4.post3762.dist-info/WHEEL,sha256=1tXe9gY0PYatrMPMDd6jXqjfpz_B-Wqm32CPfRC58XU,91
+nucliadb-6.3.4.post3762.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.4.post3762.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.4.post3762.dist-info/RECORD,,