nucliadb 6.2.1.post2903__py3-none-any.whl → 6.2.1.post2907__py3-none-any.whl

nucliadb/ingest/orm/processor/data_augmentation.py

@@ -20,13 +20,13 @@
 
 import logging
 from dataclasses import dataclass, field
-from typing import Optional, Sequence
+from typing import Optional
 
 from nucliadb.ingest.orm.resource import Resource
 from nucliadb.ingest.processing import ProcessingEngine, PushPayload, Source
 from nucliadb_models.text import PushTextFormat, Text
 from nucliadb_protos import resources_pb2, writer_pb2
-from nucliadb_protos.resources_pb2 import FieldID, FieldType
+from nucliadb_protos.resources_pb2 import FieldType
 from nucliadb_utils.utilities import Utility, get_partitioning, get_utility
 
 logger = logging.getLogger("ingest-processor")
@@ -50,7 +50,7 @@ async def get_generated_fields(bm: writer_pb2.BrokerMessage, resource: Resource)
     ingest the processed thing later).
 
     Given a broker message and a resource, this function returns the list of
-    generated fields, that can be empty.
+    generated fields, that can be empty. It skips fields with errors.
 
     """
     generated_fields = GeneratedFields()
@@ -60,34 +60,12 @@ async def get_generated_fields(bm: writer_pb2.BrokerMessage, resource: Resource)
         return generated_fields
 
     # search all fields
-
-    all_fields = await resource.get_all_field_ids(for_update=False)
-    fields: Sequence[FieldID]
-    if all_fields is None:
-        fields = []
-    else:
-        fields = all_fields.fields
-
-    for field_id in bm.texts:
-        field = FieldID(field_type=FieldType.TEXT, field=field_id)
-        if field not in fields:
+    for field_id, text in bm.texts.items():
+        errors = [e for e in bm.errors if e.field_type == FieldType.TEXT and e.field == field_id]
+        has_error = len(errors) > 0
+        if text.generated_by.WhichOneof("author") == "data_augmentation" and not has_error:
             generated_fields.texts.append(field_id)
 
-    for field_id in bm.links:
-        field = FieldID(field_type=FieldType.LINK, field=field_id)
-        if field not in fields:
-            generated_fields.links.append(field_id)
-
-    for field_id in bm.files:
-        field = FieldID(field_type=FieldType.FILE, field=field_id)
-        if field not in fields:
-            generated_fields.files.append(field_id)
-
-    for field_id in bm.conversations:
-        field = FieldID(field_type=FieldType.CONVERSATION, field=field_id)
-        if field not in fields:
-            generated_fields.conversations.append(field_id)
-
     return generated_fields
 
 

{nucliadb-6.2.1.post2903.dist-info → nucliadb-6.2.1.post2907.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.1.post2903
+Version: 6.2.1.post2907
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2903
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2903
-Requires-Dist: nucliadb-protos>=6.2.1.post2903
-Requires-Dist: nucliadb-models>=6.2.1.post2903
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post2907
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post2907
+Requires-Dist: nucliadb-protos>=6.2.1.post2907
+Requires-Dist: nucliadb-models>=6.2.1.post2907
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: nuclia-models>=0.24.2

{nucliadb-6.2.1.post2903.dist-info → nucliadb-6.2.1.post2907.dist-info}/RECORD

@@ -146,7 +146,7 @@ nucliadb/ingest/orm/resource.py,sha256=KDTEwZ6_5eLvLu1s30Pln4BFDOy9D7_ChT7kRmQ1J
 nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
-nucliadb/ingest/orm/processor/data_augmentation.py,sha256=ZF66gfHHMHCh9n9XXoTSdTXhETnzJlcylfQqhFq54Pw,5775
+nucliadb/ingest/orm/processor/data_augmentation.py,sha256=HpSU9olDHcTfECDYCsmm4yA-Hu0mBrd_zTtx50XDGFE,5164
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=f32PIEXWktWzGDws6Ffife37OAfrseP5IOti_Cb4ir8,3012
 nucliadb/ingest/orm/processor/processor.py,sha256=2FxAetUvtHvg6l-24xYrmBdsyqc0RU1zmliel44581g,30945
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
@@ -340,9 +340,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.2.1.post2903.dist-info/METADATA,sha256=x7_2u-sb13MJihkoL02nubuL4Ylypa3GhyDZbboT71k,4689
-nucliadb-6.2.1.post2903.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-nucliadb-6.2.1.post2903.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.2.1.post2903.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.2.1.post2903.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.2.1.post2903.dist-info/RECORD,,
+nucliadb-6.2.1.post2907.dist-info/METADATA,sha256=YGARv1GE9A7A4HDijyWKA0N0Rgv369hW5uprwt8lfoA,4689
+nucliadb-6.2.1.post2907.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+nucliadb-6.2.1.post2907.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.2.1.post2907.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.2.1.post2907.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.2.1.post2907.dist-info/RECORD,,