nucliadb 6.1.0.post2504__py3-none-any.whl → 6.1.0.post2517__py3-none-any.whl

nucliadb/ingest/consumer/consumer.py

@@ -291,56 +291,7 @@ class IngestProcessedConsumer(IngestConsumer):
     other writes are going to be coming from user actions and we don't want to slow them down.
     """
 
-    async def get_stream_last_seqid(self) -> Optional[int]:
-        """
-        XXX NOTE: Getting the last sequence id of the stream for the processed pull consumer is only needed when the new pull consumer is created.
-
-        For environments where we had the previous push consumer, we need to get the last sequence id from the old consumer info so that
-        we can start from there.
-
-        This code must be deleted once the new pull consumers are deployed on all environments and all the old push consumers have been deleted.
-        From then on, we will rely on the nats server to keep track of the last sequence id.
-        """
-        if has_feature(const.Features.PULL_PROCESSED_CONSUMERS_DEPLOYED):
-            logger.warning(
-                f"Feature flag {const.Features.PULL_PROCESSED_CONSUMERS_DEPLOYED} is enabled. Relying on nats to keep track of the last sequence id."
-            )
-            return None
-
-        try:
-            push_consumer_info: nats.js.api.ConsumerInfo = (
-                await self.nats_connection_manager.js.consumer_info(
-                    stream=const.Streams.INGEST_PROCESSED.name,
-                    # This is the old consumer name
-                    consumer="nucliadb-processed",
-                    timeout=2,
-                )
-            )
-            # Start from the last non-acked message
-            if push_consumer_info.ack_floor is None:
-                logger.warning(
-                    f"Nats consumer {push_consumer_info.name} has no ack floor. Starting from scratch."
-                )
-                return 1
-            last_seqid = push_consumer_info.ack_floor.stream_seq + 1
-            logger.info(
-                f"Starting from last sequence id {last_seqid} for {const.Streams.INGEST_PROCESSED.name}"
-            )
-            return last_seqid
-        except (nats.js.errors.NotFoundError, nats.errors.TimeoutError):
-            logger.warning(
-                f"Could not get last sequence id for {const.Streams.INGEST_PROCESSED.name}. Starting from scratch."
-            )
-            # Start from scratch
-            return 1
-
     async def setup_nats_subscription(self):
-        last_sequence_id = await self.get_stream_last_seqid()
-        if last_sequence_id is None:
-            delivery_policy = nats.js.api.DeliverPolicy.ALL
-        else:
-            delivery_policy = nats.js.api.DeliverPolicy.BY_START_SEQUENCE
-
         subject = const.Streams.INGEST_PROCESSED.subject
         durable_name = const.Streams.INGEST_PROCESSED.group
         self.subscription = await self.nats_connection_manager.pull_subscribe(
@@ -352,8 +303,7 @@ class IngestProcessedConsumer(IngestConsumer):
             config=nats.js.api.ConsumerConfig(
                 durable_name=durable_name,
                 ack_policy=nats.js.api.AckPolicy.EXPLICIT,
-                deliver_policy=delivery_policy,
-                opt_start_seq=last_sequence_id,
+                deliver_policy=nats.js.api.DeliverPolicy.ALL,
                 max_ack_pending=1,
                 max_deliver=nats_consumer_settings.nats_max_deliver,
                 ack_wait=nats_consumer_settings.nats_ack_wait,

nucliadb/search/search/chat/images.py

@@ -18,11 +18,14 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 import base64
+from io import BytesIO
 from typing import Optional
 
 from nucliadb.common.ids import ParagraphId
+from nucliadb.ingest.fields.file import File
 from nucliadb.search import SERVICE_NAME
 from nucliadb_models.search import Image
+from nucliadb_utils.storages.storage import Storage
 from nucliadb_utils.utilities import get_storage
 
 
@@ -64,14 +67,18 @@ async def get_paragraph_image(kbid: str, paragraph_id: ParagraphId, reference: s
     return image
 
 
-async def get_file_image(kbid: str, rid: str, field_id: str) -> Optional[Image]:
-    storage = await get_storage(service_name=SERVICE_NAME)
-    sf = storage.file_field(kbid, rid, field_id)
-    image_bytes = (await sf.storage.downloadbytes(sf.bucket, sf.key)).read()
-    if not image_bytes:
+async def get_file_thumbnail_image(file: File) -> Optional[Image]:
+    fed = await file.get_file_extracted_data()
+    if fed is None or not fed.HasField("file_thumbnail"):
+        return None
+    storage: Storage = await get_storage(service_name=SERVICE_NAME)
+    image_bytes: BytesIO = await storage.downloadbytescf(fed.file_thumbnail)
+    value = image_bytes.getvalue()
+    if len(value) == 0:
         return None
     image = Image(
-        b64encoded=base64.b64encode(image_bytes).decode(),
-        content_type="image/png",
+        b64encoded=base64.b64encode(value).decode(),
+        # We assume the thumbnail is always generated as jpeg by Nuclia processing
+        content_type="image/jpeg",
     )
     return image

nucliadb/search/search/chat/prompt.py

@@ -35,7 +35,7 @@ from nucliadb.ingest.orm.knowledgebox import KnowledgeBox as KnowledgeBoxORM
 from nucliadb.search import logger
 from nucliadb.search.search import cache
 from nucliadb.search.search.chat.images import (
-    get_file_image,
+    get_file_thumbnail_image,
     get_page_image,
     get_paragraph_image,
 )
@@ -753,14 +753,10 @@ async def conversation_prompt_context(
                         file_field: File = await resource.get_field(
                             attachment.field_id, attachment.field_type, load=True
                         )  # type: ignore
-                        field_metadata = await file_field.get_field_metadata()
-                        if field_metadata is not None and field_metadata.metadata.mime_type.startswith(
-                            "image"
-                        ):
-                            image = await get_file_image(kbid, rid, attachment.field_id)
-                            if image is not None:
-                                pid = f"{rid}/f/{attachment.field_id}/0-0"
-                                context.images[pid] = image
+                        image = await get_file_thumbnail_image(file_field)
+                        if image is not None:
+                            pid = f"{rid}/f/{attachment.field_id}/0-0"
+                            context.images[pid] = image
 
                 analyzed_fields.append(field_unique_id)
 

{nucliadb-6.1.0.post2504.dist-info → nucliadb-6.1.0.post2517.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: nucliadb
-Version: 6.1.0.post2504
+Version: 6.1.0.post2517
 Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
 Author: NucliaDB Community
 Author-email: nucliadb@nuclia.com
@@ -22,10 +22,10 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: >=3.9, <4
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.1.0.post2504
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.1.0.post2504
-Requires-Dist: nucliadb-protos>=6.1.0.post2504
-Requires-Dist: nucliadb-models>=6.1.0.post2504
+Requires-Dist: nucliadb-telemetry[all]>=6.1.0.post2517
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.1.0.post2517
+Requires-Dist: nucliadb-protos>=6.1.0.post2517
+Requires-Dist: nucliadb-models>=6.1.0.post2517
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: uvicorn

{nucliadb-6.1.0.post2504.dist-info → nucliadb-6.1.0.post2517.dist-info}/RECORD

@@ -115,7 +115,7 @@ nucliadb/ingest/settings.py,sha256=SDQpMRsTsNyi6IDxCJy6BZVUSKUzwAMuxf6ktp31VMM,3
 nucliadb/ingest/utils.py,sha256=l1myURu3r8oA11dx3GpHw-gNTUc1AFX8xdPm9Lgl2rA,2275
 nucliadb/ingest/consumer/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/consumer/auditing.py,sha256=EJoqRRr4dk2eUMK0GOY6b9xHO0YLQ0LjoP_xZBLACZo,7280
-nucliadb/ingest/consumer/consumer.py,sha256=G4YIDSTqbZALMf9COAeitPrFmDbxNUvmBMIC0h_bUvo,16341
+nucliadb/ingest/consumer/consumer.py,sha256=y9XF4U3cZN5JFzC-ZnBCn7z5viJDMmjoeEprxm7VghE,13941
 nucliadb/ingest/consumer/materializer.py,sha256=7ofLbwjldJA8TWXDRZRM4U5EviZt3qNSQ8oadmkzS0Y,3840
 nucliadb/ingest/consumer/metrics.py,sha256=ji1l_4cKiHJthQd8YNem1ft4iMbw9KThmVvJmLcv3Xg,1075
 nucliadb/ingest/consumer/pull.py,sha256=EYT0ImngMQgatStG68p2GSrPQBbJxeuq8nFm8DdAbwk,9280
@@ -229,8 +229,8 @@ nucliadb/search/search/utils.py,sha256=iF2tbBA56gRMJH1TlE2hMrqeXqjoeOPt4KgRdp2m9
 nucliadb/search/search/chat/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/chat/ask.py,sha256=2ZIz3TbCG18X0eJZSfLc8nHyKToyrlv_SPFzOOcc5wU,33865
 nucliadb/search/search/chat/exceptions.py,sha256=Siy4GXW2L7oPhIR86H3WHBhE9lkV4A4YaAszuGGUf54,1356
-nucliadb/search/search/chat/images.py,sha256=2o5zimZF_YMa8Kkbugc5BxCBdmkCSILQuHHEEt49ilo,2791
-nucliadb/search/search/chat/prompt.py,sha256=ZQZZcrp911KKHmBOWfMRb4klbAHxn_bQl7nmVu74uws,46414
+nucliadb/search/search/chat/images.py,sha256=PA8VWxT5_HUGfW1ULhKTK46UBsVyINtWWqEM1ulzX1E,3095
+nucliadb/search/search/chat/prompt.py,sha256=mreHIjhIWCIGDlJ2CA7kv7KLgh8d09meoMX3-bvWje8,46152
 nucliadb/search/search/chat/query.py,sha256=gKtlj2ms81m417Id29-DtHFxE3M4TtJvYNB03gAgpYo,14402
 nucliadb/search/search/query_parser/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/search/search/query_parser/exceptions.py,sha256=tuzl7ZyvVsRz6u0_3zMe60vx39nd3pi641prs-5nC0E,872
@@ -331,9 +331,9 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.1.0.post2504.dist-info/METADATA,sha256=vXm2SKesdaakWovns1MAt3f7inEItknaZKsAyBlkXTY,4390
-nucliadb-6.1.0.post2504.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
-nucliadb-6.1.0.post2504.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.1.0.post2504.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.1.0.post2504.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
-nucliadb-6.1.0.post2504.dist-info/RECORD,,
+nucliadb-6.1.0.post2517.dist-info/METADATA,sha256=UFOaIFgplfz5-oLinOoDftmNMSzWFqagyIpXH3eCuwo,4390
+nucliadb-6.1.0.post2517.dist-info/WHEEL,sha256=PZUExdf71Ui_so67QXpySuHtCi3-J3wvF4ORK6k_S8U,91
+nucliadb-6.1.0.post2517.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.1.0.post2517.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.1.0.post2517.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
+nucliadb-6.1.0.post2517.dist-info/RECORD,,