nucliadb 6.9.0.post5008__py3-none-any.whl → 6.9.0.post5016__py3-none-any.whl

nucliadb/ingest/orm/index_message.py

@@ -20,7 +20,7 @@
 
 
 import asyncio
-from typing import Optional
+from typing import Optional, Sequence
 
 from nidx_protos.noderesources_pb2 import Resource as IndexMessage
 
@@ -150,7 +150,7 @@ class IndexMessageBuilder:
     def _apply_field_deletions(
         self,
         brain: ResourceBrain,
-        field_ids: list[FieldID],
+        field_ids: Sequence[FieldID],
     ) -> None:
         for field_id in field_ids:
             brain.delete_field(self.resource.generate_field_id(field_id))
@@ -158,20 +158,19 @@ class IndexMessageBuilder:
     @observer.wrap({"type": "writer_bm"})
     async def for_writer_bm(
         self,
-        messages: list[BrokerMessage],
+        message: BrokerMessage,
         resource_created: bool,
     ) -> IndexMessage:
         """
-        Builds the index message for the broker messages coming from the writer.
+        Builds the index message for the broker message coming from the writer.
         The writer messages are not adding new vectors to the index.
         """
-        assert all(message.source == BrokerMessage.MessageSource.WRITER for message in messages)
+        assert message.source == BrokerMessage.MessageSource.WRITER
 
-        deleted_fields = get_bm_deleted_fields(messages)
-        self._apply_field_deletions(self.brain, deleted_fields)
+        self._apply_field_deletions(self.brain, message.delete_fields)
         await self._apply_resource_index_data(self.brain)
         basic = await self.get_basic()
-        prefilter_update = needs_prefilter_update(messages)
+        prefilter_update = needs_prefilter_update(message)
         if prefilter_update:
             # Changes on some metadata at the resource level that is used for filtering require that we reindex all the fields
             # in the texts index (as it is the one used for prefiltering).
@@ -181,16 +180,16 @@ class IndexMessageBuilder:
             ]
         else:
             # Simply process the fields that are in the message
-            fields_to_index = get_bm_modified_fields(messages)
+            fields_to_index = get_bm_modified_fields(message)
         for fieldid in fields_to_index:
-            if fieldid in deleted_fields:
+            if fieldid in message.delete_fields:
                 continue
             await self._apply_field_index_data(
                 self.brain,
                 fieldid,
                 basic,
-                texts=prefilter_update or needs_texts_update(fieldid, messages),
-                paragraphs=needs_paragraphs_update(fieldid, messages),
+                texts=prefilter_update or needs_texts_update(fieldid, message),
+                paragraphs=needs_paragraphs_update(fieldid, message),
                 relations=False,  # Relations at the field level are not modified by the writer
                 vectors=False,  # Vectors are never added by the writer
                 replace=not resource_created,
@@ -200,30 +199,29 @@ class IndexMessageBuilder:
     @observer.wrap({"type": "processor_bm"})
     async def for_processor_bm(
         self,
-        messages: list[BrokerMessage],
+        message: BrokerMessage,
     ) -> IndexMessage:
         """
         Builds the index message for the broker messages coming from the processor.
         The processor can index new data to any index.
         """
-        assert all(message.source == BrokerMessage.MessageSource.PROCESSOR for message in messages)
-        deleted_fields = get_bm_deleted_fields(messages)
-        self._apply_field_deletions(self.brain, deleted_fields)
+        assert message.source == BrokerMessage.MessageSource.PROCESSOR
+        self._apply_field_deletions(self.brain, message.delete_fields)
         await self._apply_resource_index_data(self.brain)
         basic = await self.get_basic()
-        fields_to_index = get_bm_modified_fields(messages)
+        fields_to_index = get_bm_modified_fields(message)
         vectorsets_configs = await self.get_vectorsets_configs()
         for fieldid in fields_to_index:
-            if fieldid in deleted_fields:
+            if fieldid in message.delete_fields:
                 continue
             await self._apply_field_index_data(
                 self.brain,
                 fieldid,
                 basic,
-                texts=needs_texts_update(fieldid, messages),
-                paragraphs=needs_paragraphs_update(fieldid, messages),
-                relations=needs_relations_update(fieldid, messages),
-                vectors=needs_vectors_update(fieldid, messages),
+                texts=needs_texts_update(fieldid, message),
+                paragraphs=needs_paragraphs_update(fieldid, message),
+                relations=needs_relations_update(fieldid, message),
+                vectors=needs_vectors_update(fieldid, message),
                 replace=True,
                 vectorset_configs=vectorsets_configs,
             )
@@ -270,130 +268,97 @@ class IndexMessageBuilder:
         return vectorset_configs
 
 
-def get_bm_deleted_fields(
-    messages: list[BrokerMessage],
-) -> list[FieldID]:
-    deleted = []
-    for message in messages:
-        for field in message.delete_fields:
-            if field not in deleted:
-                deleted.append(field)
-    return deleted
-
-
-def get_bm_modified_fields(messages: list[BrokerMessage]) -> list[FieldID]:
-    message_source = get_messages_source(messages)
+def get_bm_modified_fields(message: BrokerMessage) -> list[FieldID]:
     modified = set()
-    for message in messages:
-        # Added or modified fields need indexing
-        for link in message.links:
-            modified.add((link, FieldType.LINK))
-        for file in message.files:
-            modified.add((file, FieldType.FILE))
-        for conv in message.conversations:
-            modified.add((conv, FieldType.CONVERSATION))
-        for text in message.texts:
-            modified.add((text, FieldType.TEXT))
+    # Added or modified fields need indexing
+    for link in message.links:
+        modified.add((link, FieldType.LINK))
+    for file in message.files:
+        modified.add((file, FieldType.FILE))
+    for conv in message.conversations:
+        modified.add((conv, FieldType.CONVERSATION))
+    for text in message.texts:
+        modified.add((text, FieldType.TEXT))
+    if message.HasField("basic"):
+        # Add title and summary only if they have changed
+        if message.basic.title != "":
+            modified.add(("title", FieldType.GENERIC))
+        if message.basic.summary != "":
+            modified.add(("summary", FieldType.GENERIC))
+
+    if message.source == BrokerMessage.MessageSource.PROCESSOR:
+        # Messages with field metadata, extracted text or field vectors need indexing
+        for fm in message.field_metadata:
+            modified.add((fm.field.field, fm.field.field_type))
+        for et in message.extracted_text:
+            modified.add((et.field.field, et.field.field_type))
+        for fv in message.field_vectors:
+            modified.add((fv.field.field, fv.field.field_type))
+
+    if message.source == BrokerMessage.MessageSource.WRITER:
+        # Any field that has fieldmetadata annotations should be considered as modified
+        # and needs to be reindexed
         if message.HasField("basic"):
-            # Add title and summary only if they have changed
-            if message.basic.title != "":
-                modified.add(("title", FieldType.GENERIC))
-            if message.basic.summary != "":
-                modified.add(("summary", FieldType.GENERIC))
-
-        if message_source == BrokerMessage.MessageSource.PROCESSOR:
-            # Messages with field metadata, extracted text or field vectors need indexing
-            for fm in message.field_metadata:
-                modified.add((fm.field.field, fm.field.field_type))
-            for et in message.extracted_text:
-                modified.add((et.field.field, et.field.field_type))
-            for fv in message.field_vectors:
-                modified.add((fv.field.field, fv.field.field_type))
-
-        if message_source == BrokerMessage.MessageSource.WRITER:
-            # Any field that has fieldmetadata annotations should be considered as modified
-            # and needs to be reindexed
-            if message.HasField("basic"):
-                for ufm in message.basic.fieldmetadata:
-                    modified.add((ufm.field.field, ufm.field.field_type))
+            for ufm in message.basic.fieldmetadata:
+                modified.add((ufm.field.field, ufm.field.field_type))
     return [FieldID(field=field, field_type=field_type) for field, field_type in modified]
 
 
-def get_messages_source(messages: list[BrokerMessage]) -> BrokerMessage.MessageSource.ValueType:
-    assert len(set(message.source for message in messages)) == 1
-    return messages[0].source
-
-
-def needs_prefilter_update(messages: list[BrokerMessage]) -> bool:
-    return any(message.reindex for message in messages)
+def needs_prefilter_update(message: BrokerMessage) -> bool:
+    return message.reindex
 
 
-def needs_paragraphs_update(field_id: FieldID, messages: list[BrokerMessage]) -> bool:
+def needs_paragraphs_update(field_id: FieldID, message: BrokerMessage) -> bool:
     return (
-        has_paragraph_annotations(field_id, messages)
-        or has_new_extracted_text(field_id, messages)
-        or has_new_field_metadata(field_id, messages)
+        has_paragraph_annotations(field_id, message)
+        or has_new_extracted_text(field_id, message)
+        or has_new_field_metadata(field_id, message)
     )
 
 
-def has_paragraph_annotations(field_id: FieldID, messages: list[BrokerMessage]) -> bool:
-    for message in messages:
-        ufm = next(
-            (fm for fm in message.basic.fieldmetadata if fm.field == field_id),
-            None,
-        )
-        if ufm is None:
-            continue
-        if len(ufm.paragraphs) > 0:
-            return True
-    return False
+def has_paragraph_annotations(field_id: FieldID, message: BrokerMessage) -> bool:
+    ufm = next(
+        (fm for fm in message.basic.fieldmetadata if fm.field == field_id),
+        None,
+    )
+    if ufm is None:
+        return False
+    return len(ufm.paragraphs) > 0
 
 
 def has_new_field_metadata(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    for message in messages:
-        for field_metadata in message.field_metadata:
-            if field_metadata.field == field_id:
-                return True
-    return False
+    return any(field_metadata.field == field_id for field_metadata in message.field_metadata)
 
 
 def has_new_extracted_text(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    for message in messages:
-        for extracted_text in message.extracted_text:
-            if extracted_text.field == field_id:
-                return True
-    return False
+    return any(extracted_text.field == field_id for extracted_text in message.extracted_text)
 
 
 def needs_texts_update(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    return has_new_extracted_text(field_id, messages) or has_new_field_metadata(field_id, messages)
+    return has_new_extracted_text(field_id, message) or has_new_field_metadata(field_id, message)
 
 
 def needs_vectors_update(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    for message in messages:
-        for field_vectors in message.field_vectors:
-            if field_vectors.field == field_id:
-                return True
-    return False
+    return any(field_vectors.field == field_id for field_vectors in message.field_vectors)
 
 
 def needs_relations_update(
     field_id: FieldID,
-    messages: list[BrokerMessage],
+    message: BrokerMessage,
 ) -> bool:
-    return has_new_field_metadata(field_id, messages) or has_new_extracted_text(field_id, messages)
+    return has_new_field_metadata(field_id, message) or has_new_extracted_text(field_id, message)
 
 
 async def get_resource_index_message(

nucliadb/ingest/orm/processor/processor.py

@@ -141,8 +141,6 @@ class Processor:
     and can not use the txn id
     """
 
-    messages: dict[str, list[writer_pb2.BrokerMessage]]
-
     def __init__(
         self,
         driver: Driver,
@@ -150,7 +148,6 @@ class Processor:
         pubsub: Optional[PubSubDriver] = None,
         partition: Optional[str] = None,
     ):
-        self.messages = {}
         self.driver = driver
         self.storage = storage
         self.partition = partition
@@ -179,18 +176,12 @@ class Processor:
         if message.type == writer_pb2.BrokerMessage.MessageType.DELETE:
             await self.delete_resource(message, seqid, partition, transaction_check)
         elif message.type == writer_pb2.BrokerMessage.MessageType.AUTOCOMMIT:
-            await self.txn([message], seqid, partition, transaction_check)
-        elif message.type == writer_pb2.BrokerMessage.MessageType.MULTI:
-            # XXX Not supported right now
-            # MULTI, COMMIT and ROLLBACK are all not supported in transactional mode right now
-            # This concept is probably not tenable with current architecture because
-            # of how nats works and how we would need to manage rollbacks.
-            # XXX Should this be removed?
-            await self.multi(message, seqid)
-        elif message.type == writer_pb2.BrokerMessage.MessageType.COMMIT:
-            await self.commit(message, seqid, partition)
-        elif message.type == writer_pb2.BrokerMessage.MessageType.ROLLBACK:
-            await self.rollback(message, seqid, partition)
+            await self.txn(message, seqid, partition, transaction_check)
+        else:  # pragma: no cover
+            logger.error(
+                f"Unsupported message type: {message.type}",
+                extra={"seqid": seqid, "partition": partition},
+            )
 
     async def get_resource_uuid(self, kb: KnowledgeBox, message: writer_pb2.BrokerMessage) -> str:
         if message.uuid is None:
@@ -242,7 +233,6 @@ class Processor:
                         await self.notify_abort(
                             partition=partition,
                             seqid=seqid,
-                            multi=message.multiid,
                             kbid=message.kbid,
                             rid=message.uuid,
                             source=message.source,
@@ -256,7 +246,6 @@ class Processor:
         await self.notify_commit(
             partition=partition,
             seqid=seqid,
-            multi=message.multiid,
             message=message,
             write_type=writer_pb2.Notification.WriteType.DELETED,
         )
@@ -277,15 +266,12 @@ class Processor:
     @processor_observer.wrap({"type": "txn"})
     async def txn(
         self,
-        messages: list[writer_pb2.BrokerMessage],
+        message: writer_pb2.BrokerMessage,
         seqid: int,
         partition: str,
         transaction_check: bool = True,
     ) -> None:
-        if len(messages) == 0:
-            return None
-
-        kbid = messages[0].kbid
+        kbid = message.kbid
         if not await datamanagers.atomic.kb.exists_kb(kbid=kbid):
             logger.info(f"KB {kbid} is deleted: skiping txn")
             if transaction_check:
@@ -296,58 +282,55 @@ class Processor:
 
         async with self.driver.rw_transaction() as txn:
             try:
-                multi = messages[0].multiid
                 kb = KnowledgeBox(txn, self.storage, kbid)
-                uuid = await self.get_resource_uuid(kb, messages[0])
+                uuid = await self.get_resource_uuid(kb, message)
+
                 resource: Optional[Resource] = None
                 handled_exception = None
                 created = False
 
-                for message in messages:
-                    if resource is not None:
-                        assert resource.uuid == message.uuid
-
-                    if message.source == writer_pb2.BrokerMessage.MessageSource.WRITER:
-                        resource = await kb.get(uuid)
-                        if resource is None:
-                            # It's a new resource
-                            resource = await kb.add_resource(uuid, message.slug, message.basic)
-                            created = True
-                        else:
-                            # It's an update from writer for an existing resource
-                            ...
-
-                    elif message.source == writer_pb2.BrokerMessage.MessageSource.PROCESSOR:
-                        resource = await kb.get(uuid)
-                        if resource is None:
-                            logger.info(
-                                f"Secondary message for resource {message.uuid} and resource does not exist, ignoring"
-                            )
-                            continue
-                        else:
-                            # It's an update from processor for an existing resource
-                            ...
-
-                        generated_fields = await get_generated_fields(message, resource)
-                        if generated_fields.is_not_empty():
-                            await send_generated_fields_to_process(
-                                kbid, resource, generated_fields, message
-                            )
-                            # TODO: remove this when processor sends the field set
-                            for generated_text in generated_fields.texts:
-                                message.texts[
-                                    generated_text
-                                ].generated_by.data_augmentation.SetInParent()
-
+                if message.source == writer_pb2.BrokerMessage.MessageSource.WRITER:
+                    resource = await kb.get(uuid)
+                    if resource is None:
+                        # It's a new resource
+                        resource = await kb.add_resource(uuid, message.slug, message.basic)
+                        created = True
+                    else:
+                        # It's an update from writer for an existing resource
+                        ...
+
+                elif message.source == writer_pb2.BrokerMessage.MessageSource.PROCESSOR:
+                    resource = await kb.get(uuid)
+                    if resource is None:
+                        logger.info(
+                            f"Processor message for resource received but the resource does not exist, ignoring.",
+                            extra={
+                                "kbid": kbid,
+                                "rid": uuid,
+                                "seqid": seqid,
+                            },
+                        )
+                        return None
                     else:
-                        raise InvalidBrokerMessage(f"Unknown broker message source: {message.source}")
+                        # It's an update from processor for an existing resource
+                        ...
 
-                    # apply changes from the broker message to the resource
-                    await self.apply_resource(message, resource, update=(not created))
+                    generated_fields = await get_generated_fields(message, resource)
+                    if generated_fields.is_not_empty():
+                        await send_generated_fields_to_process(kbid, resource, generated_fields, message)
+                        # TODO: remove this when processor sends the field set
+                        for generated_text in generated_fields.texts:
+                            message.texts[generated_text].generated_by.data_augmentation.SetInParent()
+
+                else:  # pragma: no cover
+                    raise InvalidBrokerMessage(f"Unknown broker message source: {message.source}")
+
+                # apply changes from the broker message to the resource
+                await self.apply_resource(message, resource, update=(not created))
 
                 # index message
                 if resource and resource.modified:
-                    index_message = await self.generate_index_message(resource, messages, created)
+                    index_message = await self.generate_index_message(resource, message, created)
                     try:
                         warnings = await self.index_resource(
                             index_message=index_message,
@@ -357,7 +340,7 @@ class Processor:
                             seqid=seqid,
                             partition=partition,
                             kb=kb,
-                            source=messages_source(messages),
+                            source=to_index_message_source(message),
                         )
                         # Save indexing warnings
                         for field_id, warning in warnings:
@@ -385,7 +368,6 @@ class Processor:
                     await self.notify_commit(
                         partition=partition,
                         seqid=seqid,
-                        multi=multi,
                         message=message,
                         write_type=(
                             writer_pb2.Notification.WriteType.CREATED
@@ -398,7 +380,6 @@ class Processor:
                     await self.notify_abort(
                         partition=partition,
                         seqid=seqid,
-                        multi=multi,
                         kbid=kbid,
                         rid=uuid,
                         source=message.source,
@@ -418,7 +399,6 @@ class Processor:
                 await self.notify_abort(
                     partition=partition,
                     seqid=seqid,
-                    multi=multi,
                     kbid=kbid,
                     rid=uuid,
                     source=message.source,
@@ -428,11 +408,10 @@ class Processor:
                 # As we are in the middle of a transaction, we cannot let the exception raise directly
                 # as we need to do some cleanup. The exception will be reraised at the end of the function
                 # and then handled by the top caller, so errors can be handled in the same place.
-                await self.deadletter(messages, partition, seqid)
+                await self.deadletter(message, partition, seqid)
                 await self.notify_abort(
                     partition=partition,
                     seqid=seqid,
-                    multi=multi,
                     kbid=kbid,
                     rid=uuid,
                     source=message.source,
@@ -518,17 +497,16 @@ class Processor:
     async def generate_index_message(
         self,
         resource: Resource,
-        messages: list[writer_pb2.BrokerMessage],
+        message: writer_pb2.BrokerMessage,
         resource_created: bool,
     ) -> PBBrainResource:
         builder = IndexMessageBuilder(resource)
-        message_source = messages_source(messages)
-        if message_source == nodewriter_pb2.IndexMessageSource.WRITER:
-            return await builder.for_writer_bm(messages, resource_created)
-        elif message_source == nodewriter_pb2.IndexMessageSource.PROCESSOR:
-            return await builder.for_processor_bm(messages)
+        if message.source == writer_pb2.BrokerMessage.MessageSource.WRITER:
+            return await builder.for_writer_bm(message, resource_created)
+        elif message.source == writer_pb2.BrokerMessage.MessageSource.PROCESSOR:
+            return await builder.for_processor_bm(message)
         else:  # pragma: no cover
-            raise InvalidBrokerMessage(f"Unknown broker message source: {message_source}")
+            raise InvalidBrokerMessage(f"Unknown broker message source: {message.source}")
 
     async def external_index_delete_resource(
         self, external_index_manager: ExternalIndexManager, resource_uuid: str
@@ -581,35 +559,8 @@ class Processor:
             resource_uuid=resource_uuid, resource_data=index_message
         )
 
-    async def multi(self, message: writer_pb2.BrokerMessage, seqid: int) -> None:
-        self.messages.setdefault(message.multiid, []).append(message)
-
-    async def commit(self, message: writer_pb2.BrokerMessage, seqid: int, partition: str) -> None:
-        if message.multiid not in self.messages:
-            # Error
-            logger.error(f"Closed multi {message.multiid}")
-            await self.deadletter([message], partition, seqid)
-        else:
-            await self.txn(self.messages[message.multiid], seqid, partition)
-
-    async def rollback(self, message: writer_pb2.BrokerMessage, seqid: int, partition: str) -> None:
-        # Error
-        logger.error(f"Closed multi {message.multiid}")
-        del self.messages[message.multiid]
-        await self.notify_abort(
-            partition=partition,
-            seqid=seqid,
-            multi=message.multiid,
-            kbid=message.kbid,
-            rid=message.uuid,
-            source=message.source,
-        )
-
-    async def deadletter(
-        self, messages: list[writer_pb2.BrokerMessage], partition: str, seqid: int
-    ) -> None:
-        for seq, message in enumerate(messages):
-            await self.storage.deadletter(message, seq, seqid, partition)
+    async def deadletter(self, message: writer_pb2.BrokerMessage, partition: str, seqid: int) -> None:
+        await self.storage.deadletter(message, 0, seqid, partition)
 
     @processor_observer.wrap({"type": "apply_resource"})
     async def apply_resource(
@@ -669,7 +620,6 @@ class Processor:
         *,
         partition: str,
         seqid: int,
-        multi: str,
         message: writer_pb2.BrokerMessage,
         write_type: writer_pb2.Notification.WriteType.ValueType,
     ):
@@ -677,7 +627,7 @@ class Processor:
         notification = writer_pb2.Notification(
             partition=int(partition),
             seqid=seqid,
-            multi=multi,
+            multi="",
             uuid=message.uuid,
             kbid=message.kbid,
             action=writer_pb2.Notification.Action.COMMIT,
@@ -697,7 +647,6 @@ class Processor:
         *,
         partition: str,
         seqid: int,
-        multi: str,
         kbid: str,
         rid: str,
         source: writer_pb2.BrokerMessage.MessageSource.ValueType,
@@ -705,7 +654,7 @@ class Processor:
         message = writer_pb2.Notification(
             partition=int(partition),
             seqid=seqid,
-            multi=multi,
+            multi="",
             uuid=rid,
             kbid=kbid,
             action=writer_pb2.Notification.ABORT,
@@ -758,23 +707,16 @@ class Processor:
         return kbobj
 
 
-def messages_source(messages: list[writer_pb2.BrokerMessage]):
-    from_writer = all(
-        (message.source == writer_pb2.BrokerMessage.MessageSource.WRITER for message in messages)
-    )
-    from_processor = all(
-        (message.source == writer_pb2.BrokerMessage.MessageSource.PROCESSOR for message in messages)
-    )
-    if from_writer:
-        source = nodewriter_pb2.IndexMessageSource.WRITER
-    elif from_processor:
-        source = nodewriter_pb2.IndexMessageSource.PROCESSOR
+def to_index_message_source(message: writer_pb2.BrokerMessage):
+    if message.source == writer_pb2.BrokerMessage.MessageSource.WRITER:
+        return nodewriter_pb2.IndexMessageSource.WRITER
+    elif message.source == writer_pb2.BrokerMessage.MessageSource.PROCESSOR:
+        return nodewriter_pb2.IndexMessageSource.PROCESSOR
     else:  # pragma: no cover
-        msg = "Processor received multiple broker messages with different sources in the same txn!"
+        msg = f"Processor received a broker message with unexpected source! {message.source}"
         logger.error(msg)
         errors.capture_exception(Exception(msg))
-        source = nodewriter_pb2.IndexMessageSource.PROCESSOR
-    return source
+        return nodewriter_pb2.IndexMessageSource.PROCESSOR
 
 
 def has_vectors_operation(index_message: PBBrainResource) -> bool:

nucliadb/search/search/hydrator/images.py

@@ -30,12 +30,16 @@ from nucliadb_protos import resources_pb2
 from nucliadb_utils.utilities import get_storage
 
 
-async def paragraph_source_image(kbid: str, paragraph: resources_pb2.Paragraph) -> Optional[Image]:
+async def paragraph_source_image(
+    kbid: str, paragraph_id: ParagraphId, paragraph: resources_pb2.Paragraph
+) -> Optional[Image]:
     """Certain paragraphs are extracted from images using techniques like OCR or
     inception. If that's the case, return the original image for this paragraph.
 
     """
     source_image = paragraph.representation.reference_file
+    if not source_image:
+        return None
 
     if paragraph.kind not in (
         resources_pb2.Paragraph.TypeParagraph.OCR,
@@ -43,7 +47,7 @@ async def paragraph_source_image(kbid: str, paragraph: resources_pb2.Paragraph)
     ):
         return None
 
-    field_id = ParagraphId.from_string(paragraph.key).field_id
+    field_id = paragraph_id.field_id
 
     # Paragraphs extracted from an image store its original image representation
     # in the reference file. The path is incomplete though, as it's stored in

nucliadb/search/search/hydrator/paragraphs.py

@@ -219,7 +219,9 @@ async def hydrate_paragraph(
                 hydrated.image = hydration_models.HydratedParagraphImage()
 
                 if config.image.source_image:
-                    hydrated.image.source_image = await paragraph_source_image(kbid, paragraph)
+                    hydrated.image.source_image = await paragraph_source_image(
+                        kbid, paragraph_id, paragraph
+                    )
 
             if config.page:
                 if hydrated.page is None:

nucliadb/train/generators/sentence_classifier.py

@@ -116,10 +116,7 @@ async def get_sentences(kbid: str, result: str) -> list[str]:
     if split is not None:
         text = extracted_text.split_text[split]
         for paragraph in field_metadata.split_metadata[split].paragraphs:
-            if paragraph.key == "":
-                key = f"{rid}/{field_type}/{field}/{paragraph.start}-{paragraph.end}"
-            else:
-                key = paragraph.key
+            key = f"{rid}/{field_type}/{field}/{paragraph.start}-{paragraph.end}"
             if key == result:
                 for sentence in paragraph.sentences:
                     splitted_text = text[sentence.start : sentence.end]
@@ -127,10 +124,7 @@ async def get_sentences(kbid: str, result: str) -> list[str]:
     else:
         text = extracted_text.text
         for paragraph in field_metadata.metadata.paragraphs:
-            if paragraph.key == "":
-                key = f"{rid}/{field_type}/{field}/{paragraph.start}-{paragraph.end}"
-            else:
-                key = paragraph.key
+            key = f"{rid}/{field_type}/{field}/{paragraph.start}-{paragraph.end}"
             if key == result:
                 for sentence in paragraph.sentences:
                     splitted_text = text[sentence.start : sentence.end]

{nucliadb-6.9.0.post5008.dist-info → nucliadb-6.9.0.post5016.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.9.0.post5008
+Version: 6.9.0.post5016
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.9.0.post5008
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.9.0.post5008
-Requires-Dist: nucliadb-protos>=6.9.0.post5008
-Requires-Dist: nucliadb-models>=6.9.0.post5008
-Requires-Dist: nidx-protos>=6.9.0.post5008
+Requires-Dist: nucliadb-telemetry[all]>=6.9.0.post5016
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.9.0.post5016
+Requires-Dist: nucliadb-protos>=6.9.0.post5016
+Requires-Dist: nucliadb-models>=6.9.0.post5016
+Requires-Dist: nidx-protos>=6.9.0.post5016
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.50.0
 Requires-Dist: uvicorn[standard]

{nucliadb-6.9.0.post5008.dist-info → nucliadb-6.9.0.post5016.dist-info}/RECORD

@@ -165,7 +165,7 @@ nucliadb/ingest/orm/brain_v2.py,sha256=8MAo1N_nhoGy73TvKBuaw-NaMxIanRPCNttw6dFY4
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=kXyeF6XOpFKhEsGLcY-GLIk21Exp0cJst4XQQ9jJoug,14791
 nucliadb/ingest/orm/exceptions.py,sha256=gsp7TtVNQPiIEh-zf_UEJClwuFU0iu-5vzj0OrKMScg,1550
-nucliadb/ingest/orm/index_message.py,sha256=DWMTHJoVamUbK8opKl5csDvxfgz7c2j7phG1Ut4yIxk,15724
+nucliadb/ingest/orm/index_message.py,sha256=DlGLuuuCsXR_rqxd6CEZMYuOx1TIiq1mR5ue114rGUk,14473
 nucliadb/ingest/orm/knowledgebox.py,sha256=OG9dmfklYf1PgTHwQd_iFZOociLEvUSMMv1ZKeUgecE,23910
 nucliadb/ingest/orm/metrics.py,sha256=OiuggTh-n3kZHA2G73NEUdIlh8c3yFrbusI88DK-Mko,1273
 nucliadb/ingest/orm/resource.py,sha256=zQeZyZ-tCxr-DhonLobfZRkz_iEew0Y-cGfXeNNIHG0,40432
@@ -173,7 +173,7 @@ nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,
 nucliadb/ingest/orm/processor/__init__.py,sha256=xhDNKCxY0XNOlIVKEtM8QT75vDUkJIt7K-_VgGbbOQU,904
 nucliadb/ingest/orm/processor/auditing.py,sha256=gxn5v30KVaH0TnIjo715mWjzKGJ-DMviElEXJG9BNN4,4612
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
-nucliadb/ingest/orm/processor/processor.py,sha256=CPOf5KZgTcKLO6LxkFia2LueskcKBlv5bTifnTOZ1KE,33845
+nucliadb/ingest/orm/processor/processor.py,sha256=3wlGiCXXaVpqLyCJ3AHWHfpKTwtywU78_LkNtpqOiuQ,31391
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=kUH0bCuM6NqpA0xSwfyb9igig3Btu57pc8VYnKggqx4,1693
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -274,8 +274,8 @@ nucliadb/search/search/chat/prompt.py,sha256=Bk69WVki5XIzXFYO6o1uQw1feHtuMwfEx_A
 nucliadb/search/search/chat/query.py,sha256=qWrwVEX_GrDV7LFRyC21BURtl-WsRt8BkIocY8njkKM,17147
 nucliadb/search/search/hydrator/__init__.py,sha256=3Pc-rcax4TI174qcrllnReE728DoJTaA8tpvBUFf98g,7005
 nucliadb/search/search/hydrator/fields.py,sha256=LhKw-aNU5eJqfZADtq3iB7AGXm0l_QabAAoSHJTk8Is,5962
-nucliadb/search/search/hydrator/images.py,sha256=qqjQbn82DgMUrNG21r8CNnpnOK4RnqHtnJ0jdQsvfyU,4550
-nucliadb/search/search/hydrator/paragraphs.py,sha256=yZXsTFAzH_c9Yf8sClLHsr2OwjUxoc2KQtUcV1RT7ug,12616
+nucliadb/search/search/hydrator/images.py,sha256=gS7-dr1e_DpRQ6XaGxwMW1AMEV9a-u73h_jTVy7XY38,4602
+nucliadb/search/search/hydrator/paragraphs.py,sha256=7JjifwhvY7V2TPSnXoIM6aBmh-i6WwTT29D5CvF2Nu0,12676
 nucliadb/search/search/hydrator/resources.py,sha256=1pNyUac8xWRnQVXU6FkDwsXHzmOZwqKRgKhGPCUdEhE,2004
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=sVl9gRNzhE-s480LBBVkiXzNRbKhYRQN5F3it5tNNp8,939
@@ -343,7 +343,7 @@ nucliadb/train/generators/image_classifier.py,sha256=46YShcl7nf1_iLXZklWTAFvUNII
 nucliadb/train/generators/paragraph_classifier.py,sha256=Jk3B8a2zkAf-2-59RECiNmNP1Nz7f0-hfnS4rIE69xA,2817
 nucliadb/train/generators/paragraph_streaming.py,sha256=axGNYjOTgxGsOcAAvCz_rTYzZCdZf0f1s-Hzn-VvIX0,3646
 nucliadb/train/generators/question_answer_streaming.py,sha256=YXjWQc_SJ_TyXXJRg8tGkC9IeIzbRDcn0spQw9AlRNY,5740
-nucliadb/train/generators/sentence_classifier.py,sha256=4JBGtInnWUQrM9wBd-P7Z2lTK1Dka6U3PPDqnLbGnFM,5177
+nucliadb/train/generators/sentence_classifier.py,sha256=izCAbjMgfH6isn48lj6z09gwCunouClqct8MuaYPwVc,4989
 nucliadb/train/generators/token_classifier.py,sha256=T8JOVR1vv5g7rn7HtcQcIZ3O5TFqh15uv5rOselLBVo,9617
 nucliadb/train/generators/utils.py,sha256=88cCuHUlOkn9UDP2NR4ru7A-5hbcLGd73sv9j0fMhmY,3590
 nucliadb/writer/__init__.py,sha256=S298mrZL3vr62OrBqi97mdLxgR5cReMlRJgnaQHZV7s,1304
@@ -384,8 +384,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vu1BGg4VqJ_x2P1u2BxqPKlSfw5orT_a3R-Ln5oPUpU,8483
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.9.0.post5008.dist-info/METADATA,sha256=lWKVwRZU7H-IiUnwX8V_02JAfoHhQ0Namxoqzqw0MDU,4158
-nucliadb-6.9.0.post5008.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.9.0.post5008.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.9.0.post5008.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.9.0.post5008.dist-info/RECORD,,
+nucliadb-6.9.0.post5016.dist-info/METADATA,sha256=rGI3PsztyvQi51WADCYUtBem7OTxjTVRTtoPRShwc2s,4158
+nucliadb-6.9.0.post5016.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.9.0.post5016.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.9.0.post5016.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.9.0.post5016.dist-info/RECORD,,