nucliadb 6.4.0.post4313__py3-none-any.whl → 6.4.0.post4317__py3-none-any.whl

nucliadb/train/generators/field_streaming.py

@@ -72,8 +72,8 @@ async def generate_field_streaming_payloads(
     for status in trainset.filter.status:
         request.filter.labels.append(f"/n/s/{status}")
 
-    total = 0
     resources = set()
+    fields = set()
 
     async for document_item in get_nidx_searcher_client().Documents(request):
         text_labels = []
@@ -81,7 +81,6 @@ async def generate_field_streaming_payloads(
             text_labels.append(label)
 
         field_id = f"{document_item.uuid}{document_item.field}"
-        total += 1
         resources.add(document_item.uuid)
 
         field_parts = document_item.field.split("/")
@@ -100,6 +99,15 @@ async def generate_field_streaming_payloads(
         tl.field_type = field_type
         tl.split = split
 
+        field_unique_key = f"{rid}/{field_type}/{field}/{split}"
+        if field_unique_key in fields:
+            # This field has already been yielded. This can happen as we are streaming directly from nidx
+            # and field deletions may not be reflected immediately in the index.
+            logger.warning(f"Duplicated field found {field_unique_key}. Skipping.", extra={"kbid": kbid})
+            continue
+
+        fields.add(field_unique_key)
+
         if trainset.exclude_text:
             tl.text.text = ""
         else:
@@ -119,11 +127,11 @@ async def generate_field_streaming_payloads(
 
         yield tl
 
-        if total % 1000 == 0:
+        if len(fields) % 1000 == 0:
             logger.info(
                 "Field streaming in progress",
                 extra={
-                    "fields": total,
+                    "fields": len(fields),
                     "resources": len(resources),
                     "kbid": kbid,
                     "shard_replica_id": shard_replica_id,
@@ -133,7 +141,7 @@ async def generate_field_streaming_payloads(
     logger.info(
         "Field streaming finished",
         extra={
-            "fields": total,
+            "fields": len(fields),
             "resources": len(resources),
             "kbid": kbid,
             "shard_replica_id": shard_replica_id,

{nucliadb-6.4.0.post4313.dist-info → nucliadb-6.4.0.post4317.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.4.0.post4313
+Version: 6.4.0.post4317
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4313
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4313
-Requires-Dist: nucliadb-protos>=6.4.0.post4313
-Requires-Dist: nucliadb-models>=6.4.0.post4313
-Requires-Dist: nidx-protos>=6.4.0.post4313
+Requires-Dist: nucliadb-telemetry[all]>=6.4.0.post4317
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.4.0.post4317
+Requires-Dist: nucliadb-protos>=6.4.0.post4317
+Requires-Dist: nucliadb-models>=6.4.0.post4317
+Requires-Dist: nidx-protos>=6.4.0.post4317
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.4.0.post4313.dist-info → nucliadb-6.4.0.post4317.dist-info}/RECORD

@@ -322,7 +322,7 @@ nucliadb/train/api/v1/shards.py,sha256=GJRnQe8P-7_VTIN1oxVmxlrDA08qVN7opEZdbF4Wx
 nucliadb/train/api/v1/trainset.py,sha256=kpnpDgiMWr1FKHZJgwH7hue5kzilA8-i9X0YHlNeHuU,2113
 nucliadb/train/generators/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/train/generators/field_classifier.py,sha256=xUA10o9CtBtilbP3uc-8Wn_zQ0oK3BrqYGqZgxh4ZLk,3428
-nucliadb/train/generators/field_streaming.py,sha256=tI6vWhfLk-AVswh7rcjcO7Gg0YzS3OKMLJJ3VhDASG0,5980
+nucliadb/train/generators/field_streaming.py,sha256=nje317SutX8QmHq-xwUphzUiozmzpCRfPXxhF_jFzdg,6441
 nucliadb/train/generators/image_classifier.py,sha256=BDXgyd5TGZRnzDnVRvp-qsRCuoTbTYwui3JiDIjuiDc,1736
 nucliadb/train/generators/paragraph_classifier.py,sha256=4sH3IQc7yJrlDs1C76SxFzL9N5mXWRZzJzoiF7y4dSQ,2703
 nucliadb/train/generators/paragraph_streaming.py,sha256=1xsc_IqP-1M0TzYTqu5qCvWBNp_J3Kyvnx8HVbToXmQ,3532
@@ -368,8 +368,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.4.0.post4313.dist-info/METADATA,sha256=-Mp65qW_udL5EUZsp6CwEysmjbvuTY3u6Qbn0Bc3epI,4223
-nucliadb-6.4.0.post4313.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
-nucliadb-6.4.0.post4313.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.4.0.post4313.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.4.0.post4313.dist-info/RECORD,,
+nucliadb-6.4.0.post4317.dist-info/METADATA,sha256=d8tAT1pIjUuErJUdwLw2yt9bgSnVJn2U7KkhldSRAZU,4223
+nucliadb-6.4.0.post4317.dist-info/WHEEL,sha256=Nw36Djuh_5VDukK0H78QzOX-_FQEo6V37m3nkm96gtU,91
+nucliadb-6.4.0.post4317.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.4.0.post4317.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.4.0.post4317.dist-info/RECORD,,