nucliadb 6.2.0.post2675__py3-none-any.whl → 6.2.1__py3-none-any.whl

nucliadb/writer/back_pressure.py

@@ -30,7 +30,6 @@ from cachetools import TTLCache
 from fastapi import HTTPException, Request
 
 from nucliadb.common import datamanagers
-from nucliadb.common.cluster.manager import get_index_nodes
 from nucliadb.common.context import ApplicationContext
 from nucliadb.common.context.fastapi import get_app_context
 from nucliadb.common.http_clients.processing import ProcessingHTTPClient
@@ -168,7 +167,7 @@ class Materializer:
         self.ingest_check_interval = ingest_check_interval
 
         self.ingest_pending: int = 0
-        self.indexing_pending: dict[str, int] = {}
+        self.indexing_pending: int = 0
 
         self._tasks: list[asyncio.Task] = []
         self._running = False
@@ -232,7 +231,7 @@ class Materializer:
         response = await self.processing_http_client.stats(kbid=kbid, timeout=0.5)
         return response.incomplete
 
-    def get_indexing_pending(self) -> dict[str, int]:
+    def get_indexing_pending(self) -> int:
         return self.indexing_pending
 
     def get_ingest_pending(self) -> int:
@@ -241,20 +240,18 @@ class Materializer:
     async def _get_indexing_pending_task(self):
         try:
             while True:
-                for node in get_index_nodes():
-                    try:
-                        with back_pressure_observer({"type": "get_indexing_pending"}):
-                            self.indexing_pending[node.id] = await get_nats_consumer_pending_messages(
-                                self.nats_manager,
-                                stream=const.Streams.INDEX.name,
-                                consumer=const.Streams.INDEX.group.format(node=node.id),
-                            )
-                    except Exception:
-                        logger.exception(
-                            "Error getting pending messages to index",
-                            exc_info=True,
-                            extra={"node_id": node.id},
+                try:
+                    with back_pressure_observer({"type": "get_indexing_pending"}):
+                        self.indexing_pending = await get_nats_consumer_pending_messages(
+                            self.nats_manager,
+                            stream="nidx",
+                            consumer="nidx",
                         )
+                except Exception:
+                    logger.exception(
+                        "Error getting pending messages to index",
+                        exc_info=True,
+                    )
                 await asyncio.sleep(self.indexing_check_interval)
         except asyncio.CancelledError:
             pass
@@ -386,7 +383,7 @@ async def check_indexing_behind(
     context: ApplicationContext,
     kbid: str,
     resource_uuid: Optional[str],
-    pending_by_node: dict[str, int],
+    pending: int,
 ):
     """
     If a resource uuid is provided, it will check the nodes that have the replicas
@@ -398,36 +395,10 @@ async def check_indexing_behind(
         # Indexing back pressure is disabled
         return
 
-    if len(pending_by_node) == 0:
-        logger.warning("No nodes found to check for pending messages")
-        return
-
-    # Get nodes that are involved in the indexing of the request
-    if resource_uuid is not None:
-        nodes_to_check = await get_nodes_for_resource_shard(context, kbid, resource_uuid)
-    else:
-        nodes_to_check = await get_nodes_for_kb_active_shards(context, kbid)
-
-    if len(nodes_to_check) == 0:
-        logger.warning(
-            "No nodes found to check for pending messages",
-            extra={"kbid": kbid, "resource_uuid": resource_uuid},
-        )
-        return
-
-    # Get the highest pending value
-    highest_pending = 0
-    for node in nodes_to_check:
-        if node not in pending_by_node:
-            logger.warning("Node not found in pending messages", extra={"node": node})
-            continue
-        if pending_by_node[node] > highest_pending:
-            highest_pending = pending_by_node[node]
-
-    if highest_pending > max_pending:
+    if pending > max_pending:
         try_after = estimate_try_after(
             rate=settings.indexing_rate,
-            pending=highest_pending,
+            pending=pending,
             max_wait=settings.max_wait_time,
         )
         data = BackPressureData(type="indexing", try_after=try_after)
@@ -437,7 +408,7 @@ async def check_indexing_behind(
                 "kbid": kbid,
                 "resource_uuid": resource_uuid,
                 "try_after": try_after,
-                "pending": highest_pending,
+                "pending": pending,
             },
         )
         raise BackPressureException(data)

nucliadb/writer/resource/basic.py

@@ -22,17 +22,19 @@ from typing import Optional
 
 from fastapi import HTTPException
 
+from nucliadb.common.models_utils import to_proto
+from nucliadb.common.models_utils.from_proto import (
+    RelationNodeTypeMap,
+    RelationTypeMap,
+)
 from nucliadb.ingest.orm.utils import set_title
 from nucliadb.ingest.processing import PushPayload
-from nucliadb_models.common import FIELD_TYPES_MAP_REVERSE
 from nucliadb_models.content_types import GENERIC_MIME_TYPE
 from nucliadb_models.file import FileField
 from nucliadb_models.link import LinkField
 from nucliadb_models.metadata import (
     ParagraphAnnotation,
     QuestionAnswerAnnotation,
-    RelationNodeTypeMap,
-    RelationTypeMap,
 )
 from nucliadb_models.text import TEXT_FORMAT_TO_MIMETYPE, PushTextFormat, Text
 from nucliadb_models.writer import (
@@ -93,7 +95,8 @@ def parse_basic_modify(bm: BrokerMessage, item: ComingResourcePayload, toprocess
         if item.metadata.language:
             bm.basic.metadata.language = item.metadata.language
         if item.metadata.languages:
-            bm.basic.metadata.languages.extend(item.metadata.languages)
+            unique_languages = list(set(item.metadata.languages))
+            bm.basic.metadata.languages.extend(unique_languages)
 
     if item.fieldmetadata is not None:
         for fieldmetadata in item.fieldmetadata:
@@ -144,9 +147,8 @@ def parse_basic_modify(bm: BrokerMessage, item: ComingResourcePayload, toprocess
                 userfieldmetadata.question_answers.append(qa_annotation_pb)
 
             userfieldmetadata.field.field = fieldmetadata.field.field
-            userfieldmetadata.field.field_type = FIELD_TYPES_MAP_REVERSE[
-                fieldmetadata.field.field_type.value
-            ]
+
+            userfieldmetadata.field.field_type = to_proto.field_type(fieldmetadata.field.field_type)
 
             bm.basic.fieldmetadata.append(userfieldmetadata)
 

nucliadb/writer/resource/field.py

@@ -23,16 +23,13 @@ from typing import Optional, Union
 from google.protobuf.json_format import MessageToDict
 
 import nucliadb_models as models
+from nucliadb.common.models_utils import from_proto, to_proto
 from nucliadb.ingest.fields.conversation import Conversation
 from nucliadb.ingest.orm.resource import Resource as ORMResource
 from nucliadb.ingest.processing import PushPayload
 from nucliadb.writer import SERVICE_NAME
 from nucliadb.writer.utilities import get_processing
-from nucliadb_models.common import (
-    FIELD_TYPES_MAP,
-    FIELD_TYPES_MAP_REVERSE,
-    FieldTypeName,
-)
+from nucliadb_models.common import FieldTypeName
 from nucliadb_models.content_types import GENERIC_MIME_TYPE
 from nucliadb_models.conversation import PushConversation
 from nucliadb_models.writer import (
@@ -40,7 +37,7 @@ from nucliadb_models.writer import (
     UpdateResourcePayload,
 )
 from nucliadb_protos import resources_pb2
-from nucliadb_protos.writer_pb2 import BrokerMessage
+from nucliadb_protos.writer_pb2 import BrokerMessage, FieldIDStatus, FieldStatus
 from nucliadb_utils.storages.storage import StorageField
 from nucliadb_utils.utilities import get_storage
 
@@ -53,6 +50,7 @@ async def extract_file_field_from_pb(field_pb: resources_pb2.FieldFile) -> str:
             language=field_pb.language,
             password=field_pb.password,
             file=models.File(payload=None, uri=field_pb.file.uri),
+            extract_strategy=field_pb.extract_strategy,
         )
         return processing.convert_external_filefield_to_str(file_field)
     else:
@@ -83,7 +81,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
     storage = await get_storage(service_name=SERVICE_NAME)
     await resource.get_fields()
     for (field_type, field_id), field in resource.fields.items():
-        field_type_name = FIELD_TYPES_MAP[field_type]
+        field_type_name = from_proto.field_type_name(field_type)
 
         if field_type_name not in {
             FieldTypeName.TEXT,
@@ -174,6 +172,8 @@ def parse_text_field(
     writer: BrokerMessage,
     toprocess: PushPayload,
 ) -> None:
+    if text_field.extract_strategy is not None:
+        writer.texts[key].extract_strategy = text_field.extract_strategy
     writer.texts[key].body = text_field.body
     writer.texts[key].format = resources_pb2.FieldText.Format.Value(text_field.format.value)
     etw = resources_pb2.ExtractedTextWrapper()
@@ -184,6 +184,13 @@ def parse_text_field(
     toprocess.textfield[key] = models.Text(
         body=text_field.body,
         format=getattr(models.PushTextFormat, text_field.format.value),
+        extract_strategy=text_field.extract_strategy,
+    )
+    writer.field_statuses.append(
+        FieldIDStatus(
+            id=resources_pb2.FieldID(field_type=resources_pb2.FieldType.TEXT, field=key),
+            status=FieldStatus.Status.PENDING,
+        )
     )
 
 
@@ -203,6 +210,13 @@ async def parse_file_field(
             key, file_field, writer, toprocess, kbid, uuid, skip_store=skip_store
         )
 
+    writer.field_statuses.append(
+        FieldIDStatus(
+            id=resources_pb2.FieldID(field_type=resources_pb2.FieldType.FILE, field=key),
+            status=FieldStatus.Status.PENDING,
+        )
+    )
+
 
 async def parse_internal_file_field(
     key: str,
@@ -216,6 +230,8 @@ async def parse_internal_file_field(
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
+    if file_field.extract_strategy is not None:
+        writer.files[key].extract_strategy = file_field.extract_strategy
 
     processing = get_processing()
 
@@ -251,6 +267,8 @@ def parse_external_file_field(
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
+    if file_field.extract_strategy is not None:
+        writer.files[key].extract_strategy = file_field.extract_strategy
     uri = file_field.file.uri
     writer.files[key].url = uri  # type: ignore
     writer.files[key].file.uri = uri  # type: ignore
@@ -293,6 +311,9 @@ def parse_link_field(
     if link_field.xpath is not None:
         writer.links[key].xpath = link_field.xpath
 
+    if link_field.extract_strategy is not None:
+        writer.links[key].extract_strategy = link_field.extract_strategy
+
     toprocess.linkfield[key] = models.LinkUpload(
         link=link_field.uri,
         headers=link_field.headers or {},
@@ -300,6 +321,13 @@ def parse_link_field(
         localstorage=link_field.localstorage or {},
         css_selector=link_field.css_selector,
         xpath=link_field.xpath,
+        extract_strategy=link_field.extract_strategy,
+    )
+    writer.field_statuses.append(
+        FieldIDStatus(
+            id=resources_pb2.FieldID(field_type=resources_pb2.FieldType.LINK, field=key),
+            status=FieldStatus.Status.PENDING,
+        )
     )
 
 
@@ -313,7 +341,6 @@ async def parse_conversation_field(
 ) -> None:
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
-
     field_value = resources_pb2.Conversation()
     convs = models.PushConversation()
     for message in conversation_field.messages:
@@ -338,7 +365,7 @@ async def parse_conversation_field(
         cm.content.attachments_fields.extend(
             [
                 resources_pb2.FieldRef(
-                    field_type=FIELD_TYPES_MAP_REVERSE[attachment.field_type],
+                    field_type=to_proto.field_type_name(attachment.field_type),
                     field_id=attachment.field_id,
                     split=attachment.split if attachment.split is not None else "",
                 )
@@ -377,3 +404,9 @@ async def parse_conversation_field(
 
     toprocess.conversationfield[key] = convs
     writer.conversations[key].CopyFrom(field_value)
+    writer.field_statuses.append(
+        FieldIDStatus(
+            id=resources_pb2.FieldID(field_type=resources_pb2.FieldType.CONVERSATION, field=key),
+            status=FieldStatus.Status.PENDING,
+        )
+    )

nucliadb/writer/settings.py

@@ -36,7 +36,7 @@ class BackPressureSettings(BaseSettings):
         alias="back_pressure_enabled",
     )
     indexing_rate: float = Field(
-        default=4,
+        default=10,
         description="Estimation of the indexing rate in messages per second. This is used to calculate the try again in time",  # noqa
     )
     ingest_rate: float = Field(
@@ -48,7 +48,7 @@ class BackPressureSettings(BaseSettings):
         description="Estimation of the processing rate in messages per second. This is used to calculate the try again in time",  # noqa
     )
     max_indexing_pending: int = Field(
-        default=200,
+        default=1000,
         description="Max number of messages pending to index in a node queue before rate limiting writes. Set to 0 to disable indexing back pressure checks",  # noqa
         alias="back_pressure_max_indexing_pending",
     )

nucliadb/writer/tus/gcs.py

@@ -354,23 +354,24 @@ class GCloudFileStorageManager(FileStorageManager):
         if dm.size == 0:
             if self.storage.session is None:
                 raise AttributeError()
-            # If there is been no size finish the upload
-            content_range = "bytes {init}-{chunk}/{total}".format(
-                init=dm.offset, chunk=dm.offset, total=dm.offset
-            )
+            # In case of empty file, we need to send a PUT request with empty body
+            # and Content-Range header set to "bytes */0"
+            headers = {
+                "Content-Length": "0",
+                "Content-Range": "bytes */0",
+            }
             resumable_uri = dm.get("resumable_uri")
             async with self.storage.session.put(
                 resumable_uri,
-                headers={
-                    "Content-Length": "0",
-                    "Content-Range": content_range,
-                },
+                headers=headers,
                 data="",
             ) as call:
-                text = await call.text()  # noqa
                 if call.status not in [200, 201, 308]:
+                    try:
+                        text = await call.text()
+                    except Exception:
+                        text = ""
                     raise GoogleCloudException(f"{call.status}: {text}")
-                return call
         path = dm.get("path")
         await dm.finish()
         return path

{nucliadb-6.2.0.post2675.dist-info → nucliadb-6.2.1.dist-info}/METADATA

@@ -1,15 +1,13 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.2
 Name: nucliadb
-Version: 6.2.0.post2675
-Home-page: https://docs.nuclia.dev/docs/management/nucliadb/intro
-Author: NucliaDB Community
-Author-email: nucliadb@nuclia.com
-License: BSD
+Version: 6.2.1
+Summary: NucliaDB
+Author-email: Nuclia <nucliadb@nuclia.com>
+License: AGPL
 Project-URL: Nuclia, https://nuclia.com
 Project-URL: Github, https://github.com/nuclia/nucliadb
 Project-URL: Slack, https://nuclia-community.slack.com
 Project-URL: API Reference, https://docs.nuclia.dev/docs/api
-Keywords: search,semantic,AI
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: Intended Audience :: Information Technology
@@ -20,14 +18,14 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
-Requires-Python: >=3.9, <4
+Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.2.0.post2675
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.0.post2675
-Requires-Dist: nucliadb-protos>=6.2.0.post2675
-Requires-Dist: nucliadb-models>=6.2.0.post2675
+Requires-Dist: nucliadb-telemetry[all]>=6.2.1.post3260
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.2.1.post3260
+Requires-Dist: nucliadb-protos>=6.2.1.post3260
+Requires-Dist: nucliadb-models>=6.2.1.post3260
+Requires-Dist: nidx-protos>=6.2.1.post3260
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
-Requires-Dist: nucliadb-node-binding>=2.26.0
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn
 Requires-Dist: argdantic
@@ -78,7 +76,6 @@ Requires-Dist: async_lru>=2.0.4
 Requires-Dist: async-timeout>=4.0.3
 Requires-Dist: cachetools>=5.3.2
 Requires-Dist: types-cachetools>=5.3.0.5
-Requires-Dist: kubernetes_asyncio<30.0.0
 Provides-Extra: redis
 Requires-Dist: redis>=4.3.4; extra == "redis"