nucliadb 6.9.0.post5068__py3-none-any.whl → 6.9.0.post5072__py3-none-any.whl

nucliadb/common/cluster/settings.py

@@ -42,7 +42,7 @@ class Settings(BaseSettings):
         description="Maximum number of paragraphs to target per shard",
     )
     max_resource_paragraphs: int = Field(
-        default=50_000,
+        default=300_000,
         title="Max paragraphs per resource",
         description="Maximum number of paragraphs allowed on a single resource",
     )

nucliadb/ingest/fields/conversation.py

@@ -25,6 +25,8 @@ from nucliadb_protos.resources_pb2 import CloudFile, FieldConversation
 from nucliadb_protos.resources_pb2 import Conversation as PBConversation
 from nucliadb_utils.storages.storage import StorageField
 
+MAX_CONVERSATION_MESSAGES = 50 * 1024
+
 PAGE_SIZE = 200
 
 CONVERSATION_PAGE_VALUE = "/kbs/{kbid}/r/{uuid}/f/{type}/{field}/{page}"

nucliadb/writer/api/v1/field.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from inspect import iscoroutinefunction
-from typing import TYPE_CHECKING, Annotated, Callable, Optional, Type, Union
+from typing import TYPE_CHECKING, Annotated, Callable, List, Optional, Type, Union
 
 import pydantic
 from fastapi import HTTPException, Query, Response
@@ -460,7 +460,7 @@ async def append_messages_to_conversation_field_rslug_prefix(
     kbid: str,
     rslug: str,
     field_id: FieldIdString,
-    messages: list[models.InputMessage],
+    messages: List[models.InputMessage],
 ) -> ResourceFieldAdded:
     try:
         field = models.InputConversationField(messages=messages)
@@ -483,7 +483,7 @@ async def append_messages_to_conversation_field_rid_prefix(
     kbid: str,
     rid: str,
     field_id: FieldIdString,
-    messages: list[models.InputMessage],
+    messages: List[models.InputMessage],
 ) -> ResourceFieldAdded:
     try:
         field = models.InputConversationField(messages=messages)

nucliadb/writer/resource/field.py

@@ -21,13 +21,14 @@ import dataclasses
 from datetime import datetime
 from typing import Optional, Union
 
+from fastapi import HTTPException
 from google.protobuf.json_format import MessageToDict
 
 import nucliadb_models as models
 from nucliadb.common import datamanagers
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.common.models_utils import from_proto, to_proto
-from nucliadb.ingest.fields.conversation import Conversation
+from nucliadb.ingest.fields.conversation import MAX_CONVERSATION_MESSAGES, Conversation
 from nucliadb.ingest.orm.resource import Resource as ORMResource
 from nucliadb.models.internal import processing as processing_models
 from nucliadb.models.internal.processing import ClassificationLabel, PushConversation, PushPayload
@@ -431,6 +432,14 @@ async def parse_conversation_field(
     uuid: str,
     resource_classifications: ResourceClassifications,
 ) -> None:
+    # Make sure that the max number of messages is not exceeded
+    current_message_count = await get_current_conversation_message_count(kbid, uuid, key)
+    if len(conversation_field.messages) + current_message_count > MAX_CONVERSATION_MESSAGES:
+        raise HTTPException(
+            status_code=422,
+            detail=f"Conversation fields cannot have more than {MAX_CONVERSATION_MESSAGES} messages.",
+        )
+
     classif_labels = resource_classifications.for_field(key, resources_pb2.FieldType.CONVERSATION)
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
@@ -543,3 +552,15 @@ async def get_stored_resource_classifications(
             classif = ClassificationLabel(labelset=f_classif.labelset, label=f_classif.label)
             rc.field_level.setdefault(fid, set()).add(classif)
     return rc
+
+
+async def get_current_conversation_message_count(kbid: str, rid: str, field_id: str) -> int:
+    async with datamanagers.with_ro_transaction() as txn:
+        resource_obj = await datamanagers.resources.get_resource(txn, kbid=kbid, rid=rid)
+        if resource_obj is None:
+            return 0
+        field_obj: Conversation = await resource_obj.get_field(
+            field_id, resources_pb2.FieldType.CONVERSATION, load=False
+        )
+        metadata = await field_obj.get_metadata()
+        return metadata.total

{nucliadb-6.9.0.post5068.dist-info → nucliadb-6.9.0.post5072.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.9.0.post5068
+Version: 6.9.0.post5072
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,11 +19,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.9.0.post5068
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.9.0.post5068
-Requires-Dist: nucliadb-protos>=6.9.0.post5068
-Requires-Dist: nucliadb-models>=6.9.0.post5068
-Requires-Dist: nidx-protos>=6.9.0.post5068
+Requires-Dist: nucliadb-telemetry[all]>=6.9.0.post5072
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.9.0.post5072
+Requires-Dist: nucliadb-protos>=6.9.0.post5072
+Requires-Dist: nucliadb-models>=6.9.0.post5072
+Requires-Dist: nidx-protos>=6.9.0.post5072
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.50.0
 Requires-Dist: uvicorn[standard]

{nucliadb-6.9.0.post5068.dist-info → nucliadb-6.9.0.post5072.dist-info}/RECORD

@@ -86,7 +86,7 @@ nucliadb/common/cluster/grpc_node_dummy.py,sha256=JkufazWzMA4KFEU8EBkMbiiDW4C8lL
 nucliadb/common/cluster/manager.py,sha256=QOHA7trtZVwV5G9IIxM1Y-0P2KE5mEhsXUMPs8vxIg0,12816
 nucliadb/common/cluster/rebalance.py,sha256=238JxqdSl1_oeJNpKuGBgfVn8WXVmyX5yl2PyHomYX8,8715
 nucliadb/common/cluster/rollover.py,sha256=swuYNXJ8u5p11PvYaHxVtomiYIPVWVGVsicwTgm24-Q,25862
-nucliadb/common/cluster/settings.py,sha256=H-KRSDdJozXLRZjjmDzgjMKm_9p6UBaPuqmTFkdhd_M,2270
+nucliadb/common/cluster/settings.py,sha256=f6Y5K0PGahkedwe5wtkWMnbqwDFJgOOwX_MOIGwH9Dg,2271
 nucliadb/common/cluster/utils.py,sha256=IfW5PA7Ab26xWUYNOc3yBoksWV1GK4BGhTRo1vnHNKg,4662
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
@@ -154,7 +154,7 @@ nucliadb/ingest/consumer/shard_creator.py,sha256=UKIk0yaS_jC_nGQqymn9NGJWzwZEqhI
 nucliadb/ingest/consumer/utils.py,sha256=jpX8D4lKzuPCpArQLZeX_Zczq3pfen_zAf8sPJfOEZU,2642
 nucliadb/ingest/fields/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/fields/base.py,sha256=D8NzawonF7hivDW9zvQBbV938TKA6e2OCGqV4kS96RU,24405
-nucliadb/ingest/fields/conversation.py,sha256=3_MZY_w0zK-9rd6Pb_sPVBUP2YB5twOZSdo_9lCa8KQ,7684
+nucliadb/ingest/fields/conversation.py,sha256=ShdPapTIG7sA05YlG1Dj2CsAwNwibsqRSQrsuZnr8YI,7723
 nucliadb/ingest/fields/exceptions.py,sha256=sZBk21BSrXFdOdo1qUdCAyD-9YMYakSLdn4_WdIPCIQ,1217
 nucliadb/ingest/fields/file.py,sha256=1v4jLg3balUua2VmSV8hHkAwPFShTUCOzufZvIUQcQw,4740
 nucliadb/ingest/fields/generic.py,sha256=elgtqv15aJUq3zY7X_g0bli_2BpcwPArVvzhe54Y4Ig,1547
@@ -360,7 +360,7 @@ nucliadb/writer/api/constants.py,sha256=SCdqGDbEmpdczQdTfbTlpHzVjbLqccPtMQ25MPIF
 nucliadb/writer/api/utils.py,sha256=wIQHlU8RQiIGVLI72suvyVIKlCU44Unh0Ae0IiN6Qwo,1313
 nucliadb/writer/api/v1/__init__.py,sha256=akI9A_jloNLb0dU4T5zjfdyvmSAiDeIdjAlzNx74FlU,1128
 nucliadb/writer/api/v1/export_import.py,sha256=v0sU55TtRSqDzwkDgcwv2uSaqKCuQTtGcMpYoHQYBQA,8192
-nucliadb/writer/api/v1/field.py,sha256=aO4GFzadXnMhGlAfvJc5KkObii_z6a8R80yCdvjKEc8,18966
+nucliadb/writer/api/v1/field.py,sha256=qcuniSwR9tR9vn5abpK3rB_olpuUTEj_0LcL_1eAiLw,18972
 nucliadb/writer/api/v1/knowledgebox.py,sha256=kioqjD3yN-y1cDTgmXAAOwivXHX9NXxwblcSzGqJup0,9533
 nucliadb/writer/api/v1/learning_config.py,sha256=DTLEzKJ3dHvi8pbZscjElUqCH_ZvLc6WZgvalFqHo10,4450
 nucliadb/writer/api/v1/resource.py,sha256=IfcT6HXnR5sC5wSnQSuKmFzEWcLTh7OzZEAV4hYmXnA,20442
@@ -373,7 +373,7 @@ nucliadb/writer/api/v1/vectorsets.py,sha256=F3iMViL5G95_Tns4aO2SOA0DwAzxK2_P8MXx
 nucliadb/writer/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/resource/audit.py,sha256=FvxMZPzrNHtd31HgpZEvxzwAkbxJTZRhPLqRYYJi3tA,1426
 nucliadb/writer/resource/basic.py,sha256=44GK8M9EEVoAUfGiabdLrrpENqeFwNn7qwxF2AHhQGg,10504
-nucliadb/writer/resource/field.py,sha256=1fHm2N-3LcgQDE3BUDiJ0ToCiwY1Di7UbA6fcJs8BIk,21193
+nucliadb/writer/resource/field.py,sha256=eM2KFxhcG3u6-ldniZDSYqGzhJ5bpWgIQBGXXFwskqw,22195
 nucliadb/writer/resource/origin.py,sha256=pvhUDdU0mlWPUcpoQi4LDUJaRtfjzVVrA8XcGVI_N8k,2021
 nucliadb/writer/tus/__init__.py,sha256=Kera0BtxoDX0ngPftXiMjNgjrhtQ3l2XFc5nJqSBOJY,5498
 nucliadb/writer/tus/azure.py,sha256=yxoRi4PhGDikTqVK3PiuVyguy8H9DOS66JpZCY4hpUY,4177
@@ -384,8 +384,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vu1BGg4VqJ_x2P1u2BxqPKlSfw5orT_a3R-Ln5oPUpU,8483
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.9.0.post5068.dist-info/METADATA,sha256=eKVcsDkBBGIK56YqrJfJepBfiOvBFf9l40ztFBAR-pA,4158
-nucliadb-6.9.0.post5068.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.9.0.post5068.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.9.0.post5068.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.9.0.post5068.dist-info/RECORD,,
+nucliadb-6.9.0.post5072.dist-info/METADATA,sha256=IuymQS1C87M_L3u-9Bpy-Wf76fYOScOwTD18Dr5srAQ,4158
+nucliadb-6.9.0.post5072.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.9.0.post5072.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.9.0.post5072.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.9.0.post5072.dist-info/RECORD,,