nucliadb 6.7.0.post4734__py3-none-any.whl → 6.7.0.post4742__py3-none-any.whl

nucliadb/common/cluster/settings.py

@@ -46,6 +46,11 @@ class Settings(BaseSettings):
         title="Max paragraphs per resource",
         description="Maximum number of paragraphs allowed on a single resource",
     )
+    max_entity_facets: int = Field(
+        default=4_000,
+        title="Max entity labels per field",
+        description="Maximum number of entity labels (/e/) per field that are indexed (excess is not indexed)",
+    )
 
     nidx_api_address: Optional[str] = Field(default=None, description="NIDX gRPC API address")
     nidx_searcher_address: Optional[str] = Field(

nucliadb/ingest/orm/exceptions.py

@@ -49,6 +49,10 @@ class ResourceNotIndexable(Exception):
     Unable to index resource
     """
 
+    def __init__(self, field_id: str, message: str):
+        self.field_id = field_id
+        self.message = message
+
 
 class EntityManagementException(Exception):
     pass

nucliadb/ingest/orm/processor/__init__.py

@@ -18,4 +18,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 # reexports
-from .processor import Processor, validate_indexable_resource  # noqa: F401
+from .processor import Processor
+
+__all__ = ["Processor"]

nucliadb/ingest/orm/processor/processor.py

@@ -83,18 +83,52 @@ def validate_indexable_resource(resource: noderesources_pb2.Resource) -> None:
     This is still an edge case.
     """
     num_paragraphs = 0
-    for _, fparagraph in resource.paragraphs.items():
+    first_exceeded = ""
+    for field_id, fparagraph in resource.paragraphs.items():
         # this count should not be very expensive to do since we don't have
         # a lot of different fields and we just do a count on a dict
         num_paragraphs += len(fparagraph.paragraphs)
+        if num_paragraphs > cluster_settings.max_resource_paragraphs:
+            first_exceeded = field_id
 
     if num_paragraphs > cluster_settings.max_resource_paragraphs:
         raise ResourceNotIndexable(
-            "Resource has too many paragraphs. "
-            f"Supported: {cluster_settings.max_resource_paragraphs} , Number: {num_paragraphs}"
+            first_exceeded,
+            f"Resource has too many paragraphs ({num_paragraphs}) and cannot be indexed. "
+            f"The maximum number of paragraphs per resource is {cluster_settings.max_resource_paragraphs}",
         )
 
 
+def trim_entity_facets(index_message: PBBrainResource) -> list[tuple[str, str]]:
+    max_entities = cluster_settings.max_entity_facets
+    warnings = []
+    for field_id, text_info in index_message.texts.items():
+        if len(text_info.labels) > max_entities:
+            new_labels = []
+            entity_count = 0
+            truncated = False
+            for label in text_info.labels:
+                if label.startswith("/e/"):
+                    if entity_count < max_entities:
+                        new_labels.append(label)
+                    else:
+                        truncated = True
+                    entity_count += 1
+                else:
+                    new_labels.append(label)
+            if truncated:
+                warnings.append(
+                    (
+                        field_id,
+                        f"Too many detected entities. Only the first {max_entities} will be available as facets for filtering",
+                    )
+                )
+                text_info.ClearField("labels")
+                text_info.labels.extend(new_labels)
+
+    return warnings
+
+
 class Processor:
     """
     This class is responsible for processing messages from the broker
@@ -314,17 +348,35 @@ class Processor:
                 # index message
                 if resource and resource.modified:
                     index_message = await self.generate_index_message(resource, messages, created)
+
+                    try:
+                        warnings = await self.index_resource(
+                            index_message=index_message,
+                            txn=txn,
+                            uuid=uuid,
+                            kbid=kbid,
+                            seqid=seqid,
+                            partition=partition,
+                            kb=kb,
+                            source=messages_source(messages),
+                        )
+                        # Save indexing warnings
+                        for field_id, warning in warnings:
+                            await resource.add_field_error(
+                                field_id, warning, writer_pb2.Error.Severity.WARNING
+                            )
+                    except ResourceNotIndexable as e:
+                        await resource.add_field_error(
+                            e.field_id, e.message, writer_pb2.Error.Severity.ERROR
+                        )
+                        # Catalog takes status from index message labels, override it to error
+                        current_status = [x for x in index_message.labels if x.startswith("/n/s/")]
+                        if current_status:
+                            index_message.labels.remove(current_status[0])
+                            index_message.labels.append("/n/s/ERROR")
+
                     await pgcatalog_update(txn, kbid, resource, index_message)
-                    await self.index_resource(  # noqa
-                        index_message=index_message,
-                        txn=txn,
-                        uuid=uuid,
-                        kbid=kbid,
-                        seqid=seqid,
-                        partition=partition,
-                        kb=kb,
-                        source=messages_source(messages),
-                    )
+
                     if transaction_check:
                         await sequence_manager.set_last_seqid(txn, partition, seqid)
                     await txn.commit()
@@ -445,8 +497,10 @@ class Processor:
         partition: str,
         kb: KnowledgeBox,
         source: nodewriter_pb2.IndexMessageSource.ValueType,
-    ) -> None:
+    ) -> list[tuple[str, str]]:
         validate_indexable_resource(index_message)
+        warnings = trim_entity_facets(index_message)
+
         shard = await self.get_or_assign_resource_shard(txn, kb, uuid)
         external_index_manager = await get_external_index_manager(kbid=kbid)
         if external_index_manager is not None:
@@ -460,6 +514,7 @@ class Processor:
                 kb=kbid,
                 source=source,
             )
+        return warnings
 
     @processor_observer.wrap({"type": "generate_index_message"})
     async def generate_index_message(

nucliadb/ingest/orm/resource.py

@@ -27,7 +27,7 @@ from typing import TYPE_CHECKING, Any, Optional, Sequence, Type
 
 from nucliadb.common import datamanagers
 from nucliadb.common.datamanagers.resources import KB_RESOURCE_SLUG
-from nucliadb.common.ids import FIELD_TYPE_PB_TO_STR, FieldId
+from nucliadb.common.ids import FIELD_TYPE_PB_TO_STR, FIELD_TYPE_STR_TO_PB, FieldId
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.ingest.fields.base import Field
 from nucliadb.ingest.fields.conversation import Conversation
@@ -526,6 +526,34 @@ class Resource:
         else:
             self.basic.metadata.status = PBMetadata.Status.PROCESSED
 
+    async def add_field_error(
+        self, field_id: str, message: str, severity: writer_pb2.Error.Severity.ValueType
+    ):
+        (field_type_str, field_name) = field_id.split("/")
+        field_type = FIELD_TYPE_STR_TO_PB[field_type_str]
+        field = await self.get_field(field_name, field_type)
+        status = await field.get_status()
+        if status is not None:
+            field_error = writer_pb2.FieldError(
+                source_error=writer_pb2.Error(
+                    field=field_name,
+                    field_type=field_type,
+                    error=message,
+                    code=writer_pb2.Error.ErrorCode.INDEX,
+                    severity=severity,
+                )
+            )
+            field_error.created.GetCurrentTime()
+            status.errors.append(field_error)
+            if severity == writer_pb2.Error.Severity.ERROR:
+                status.status = writer_pb2.FieldStatus.Status.ERROR
+            await field.set_status(status)
+
+        # If it's an error, we may need to change the resource status
+        if severity == writer_pb2.Error.Severity.ERROR and self.basic:
+            await self.update_status()
+            await self.set_basic(self.basic)
+
     @processor_observer.wrap({"type": "apply_extracted"})
     async def apply_extracted(self, message: BrokerMessage):
         await self.get_basic()

nucliadb/reader/api/v1/learning_config.py

@@ -21,7 +21,7 @@ from typing import Dict
 
 from fastapi import Request
 from fastapi_versioning import version
-from nuclia_models.config.proto import ExtractConfig
+from nuclia_models.config.proto import ExtractConfig, SplitConfiguration
 
 from nucliadb.learning_proxy import learning_config_proxy
 from nucliadb.models.responses import HTTPClientError
@@ -183,3 +183,40 @@ async def get_extract_strategy_from_id(
     return await learning_config_proxy(
         request, "GET", f"/extract_strategies/{kbid}/strategies/{strategy_id}"
     )
+
+
+@api.get(
+    path=f"/{KB_PREFIX}/{{kbid}}/split_strategies",
+    status_code=200,
+    summary="Learning split strategies",
+    description="Get available split strategies ",
+    response_model=Dict[str, SplitConfiguration],
+    tags=["Split Strategies"],
+)
+@requires(NucliaDBRoles.READER)
+@version(1)
+async def get_split_strategies(
+    request: Request,
+    kbid: str,
+):
+    return await learning_config_proxy(request, "GET", f"/split_strategies/{kbid}")
+
+
+@api.get(
+    path=f"/{KB_PREFIX}/{{kbid}}/split_strategies/strategy/{{strategy_id}}",
+    status_code=200,
+    summary="Extract split configuration",
+    description="Get split strategy for a given id",
+    response_model=None,
+    tags=["Split Strategies"],
+)
+@requires(NucliaDBRoles.READER)
+@version(1)
+async def get_split_strategy_from_id(
+    request: Request,
+    kbid: str,
+    strategy_id: str,
+):
+    return await learning_config_proxy(
+        request, "GET", f"/split_strategies/{kbid}/strategies/{strategy_id}"
+    )

nucliadb/writer/api/v1/learning_config.py

@@ -19,7 +19,7 @@
 #
 from fastapi import Request
 from fastapi_versioning import version
-from nuclia_models.config.proto import ExtractConfig
+from nuclia_models.config.proto import ExtractConfig, SplitConfiguration
 
 from nucliadb.learning_proxy import learning_config_proxy
 from nucliadb.writer.api.v1.router import KB_PREFIX, api
@@ -97,3 +97,41 @@ async def delete_strategy(
     return await learning_config_proxy(
         request, "DELETE", f"/extract_strategies/{kbid}/strategies/{strategy_id}"
     )
+
+
+@api.post(
+    path=f"/{KB_PREFIX}/{{kbid}}/split_strategies",
+    status_code=200,
+    summary="Add a split strategy to a KB",
+    description="Add a split strategy to a KB",
+    response_model=str,
+    tags=["Split Strategies"],
+)
+@requires(NucliaDBRoles.WRITER)
+@version(1)
+async def add_split_strategy(
+    request: Request,
+    kbid: str,
+    item: SplitConfiguration,
+):
+    return await learning_config_proxy(request, "POST", f"/split_strategies/{kbid}")
+
+
+@api.delete(
+    path=f"/{KB_PREFIX}/{{kbid}}/split_strategies/strategy/{{strategy_id}}",
+    status_code=204,
+    summary="Remove a split strategy from a KB",
+    description="Removes a split strategy from a KB",
+    response_model=None,
+    tags=["Split Strategies"],
+)
+@requires(NucliaDBRoles.WRITER)
+@version(1)
+async def delete_split_strategy(
+    request: Request,
+    kbid: str,
+    strategy_id: str,
+):
+    return await learning_config_proxy(
+        request, "DELETE", f"/split_strategies/{kbid}/strategies/{strategy_id}"
+    )

{nucliadb-6.7.0.post4734.dist-info → nucliadb-6.7.0.post4742.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.7.0.post4734
+Version: 6.7.0.post4742
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: AGPL-3.0-or-later
@@ -19,13 +19,13 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.7.0.post4734
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.0.post4734
-Requires-Dist: nucliadb-protos>=6.7.0.post4734
-Requires-Dist: nucliadb-models>=6.7.0.post4734
-Requires-Dist: nidx-protos>=6.7.0.post4734
+Requires-Dist: nucliadb-telemetry[all]>=6.7.0.post4742
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.7.0.post4742
+Requires-Dist: nucliadb-protos>=6.7.0.post4742
+Requires-Dist: nucliadb-models>=6.7.0.post4742
+Requires-Dist: nidx-protos>=6.7.0.post4742
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
-Requires-Dist: nuclia-models>=0.43.0
+Requires-Dist: nuclia-models>=0.46.0
 Requires-Dist: uvicorn[standard]
 Requires-Dist: argdantic
 Requires-Dist: aiohttp>=3.11.11

{nucliadb-6.7.0.post4734.dist-info → nucliadb-6.7.0.post4742.dist-info}/RECORD

@@ -81,7 +81,7 @@ nucliadb/common/cluster/grpc_node_dummy.py,sha256=JkufazWzMA4KFEU8EBkMbiiDW4C8lL
 nucliadb/common/cluster/manager.py,sha256=QOHA7trtZVwV5G9IIxM1Y-0P2KE5mEhsXUMPs8vxIg0,12816
 nucliadb/common/cluster/rebalance.py,sha256=238JxqdSl1_oeJNpKuGBgfVn8WXVmyX5yl2PyHomYX8,8715
 nucliadb/common/cluster/rollover.py,sha256=swuYNXJ8u5p11PvYaHxVtomiYIPVWVGVsicwTgm24-Q,25862
-nucliadb/common/cluster/settings.py,sha256=JPwV_0U_i618Tn66GWUq6qCKNjy4TWkGEGld9GwH5uk,2048
+nucliadb/common/cluster/settings.py,sha256=H-KRSDdJozXLRZjjmDzgjMKm_9p6UBaPuqmTFkdhd_M,2270
 nucliadb/common/cluster/utils.py,sha256=IfW5PA7Ab26xWUYNOc3yBoksWV1GK4BGhTRo1vnHNKg,4662
 nucliadb/common/cluster/standalone/__init__.py,sha256=itSI7dtTwFP55YMX4iK7JzdMHS5CQVUiB1XzQu4UBh8,833
 nucliadb/common/cluster/standalone/utils.py,sha256=af3r-x_GF7A6dwIAhZLR-r-SZQEVxsFrDKeMfUTA6G0,1908
@@ -159,17 +159,17 @@ nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20
 nucliadb/ingest/orm/brain_v2.py,sha256=y_OQ-SXyClX_VSKFVkYYwdIDxmj6TAQLC7UdPJ0hOvE,33647
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=kXyeF6XOpFKhEsGLcY-GLIk21Exp0cJst4XQQ9jJoug,14791
-nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
+nucliadb/ingest/orm/exceptions.py,sha256=gsp7TtVNQPiIEh-zf_UEJClwuFU0iu-5vzj0OrKMScg,1550
 nucliadb/ingest/orm/index_message.py,sha256=DWMTHJoVamUbK8opKl5csDvxfgz7c2j7phG1Ut4yIxk,15724
 nucliadb/ingest/orm/knowledgebox.py,sha256=_rkeTMIXMhR64gbYtZpFHoUHghV2DTJ2lUBqZsoqC_4,23898
 nucliadb/ingest/orm/metrics.py,sha256=OiuggTh-n3kZHA2G73NEUdIlh8c3yFrbusI88DK-Mko,1273
-nucliadb/ingest/orm/resource.py,sha256=MfKGHTj-hxIoSC_8_86OPfRbaQOTHGkADZIvQ_7dKW4,38780
+nucliadb/ingest/orm/resource.py,sha256=TWsPo11d6n0HpXRNGy0AAVKXiYDSr0gzEzBBQXkw7FE,40015
 nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,2693
-nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
+nucliadb/ingest/orm/processor/__init__.py,sha256=xhDNKCxY0XNOlIVKEtM8QT75vDUkJIt7K-_VgGbbOQU,904
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
 nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=VPQ_Evme7xmmGoQ45zt0Am0yPkaD4hxN1r5rEaVt6s8,4633
-nucliadb/ingest/orm/processor/processor.py,sha256=jaEBwbv--WyoC8zcdxWAyF0dAzVA5crVDJl56Bqv1eI,31444
+nucliadb/ingest/orm/processor/processor.py,sha256=pr0wbe86d6ZboOaBH5HMsgcmm-WYDIjMrGDTElPxS3A,33830
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=LHQFUkdmNBOWqBG0Md9sMMI7g5TQZ-hLAnhw6ZblrJg,2002
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -202,7 +202,7 @@ nucliadb/reader/api/v1/__init__.py,sha256=ieP8lsCCwG66Jupv8II5MSTj6nh3eCtLcF4utH
 nucliadb/reader/api/v1/download.py,sha256=rGv1c5CjrJheDgGwAnNWy76A_4V2shqqHYvwmKGFlpk,10758
 nucliadb/reader/api/v1/export_import.py,sha256=x4VBNDFjnlY1nIt5kdq0eZTB_DeRzGzT8T7uB7wUhNU,6448
 nucliadb/reader/api/v1/knowledgebox.py,sha256=Uu-yPB8KKZt1VaFrFNMMaXOvLsclBJDK9dzZ9lF2ctI,3645
-nucliadb/reader/api/v1/learning_config.py,sha256=CZ7pFXzBZkJE2dXbC1wArszJw_ZLpuEb6gnsz2MKEz0,5525
+nucliadb/reader/api/v1/learning_config.py,sha256=t_KqQBBbhpo0m6nQTkYmNdZsLVmW53SLcHMrCWiQMrk,6536
 nucliadb/reader/api/v1/resource.py,sha256=oCZGisVv5-t67MmokIpwaqJq5suE6YphR1QoCUmM174,14172
 nucliadb/reader/api/v1/router.py,sha256=eyNmEGSP9zHkCIG5XlAXl6sukq950B7gFT3X2peMtIE,1011
 nucliadb/reader/api/v1/services.py,sha256=Q2n-QMfAVzSylu6qTUuPa6L3AfZ9jS3azVlfQdSqlA4,13443
@@ -354,7 +354,7 @@ nucliadb/writer/api/v1/__init__.py,sha256=akI9A_jloNLb0dU4T5zjfdyvmSAiDeIdjAlzNx
 nucliadb/writer/api/v1/export_import.py,sha256=v0sU55TtRSqDzwkDgcwv2uSaqKCuQTtGcMpYoHQYBQA,8192
 nucliadb/writer/api/v1/field.py,sha256=rF_x6fVczZKa4xBrHChUVau-70uFJT7srQpOLuqG2ic,18755
 nucliadb/writer/api/v1/knowledgebox.py,sha256=PHEYDFa-sN5JrI8-EiVVg5FDOsRuCLT43kyAB4xt-xA,9530
-nucliadb/writer/api/v1/learning_config.py,sha256=CKBjqcbewkfPwGUPLDWzZSpro6XkmCaVppe5Qtpu5Go,3117
+nucliadb/writer/api/v1/learning_config.py,sha256=mGQmnfFSM2Z9HDnWr8PFoA1MLpFR1JaCDb8B14J8e5k,4140
 nucliadb/writer/api/v1/resource.py,sha256=gr4VtYOtnIAL5zsDQsBDzeOOWWh6wN7PUh1XgUsQ9N4,20436
 nucliadb/writer/api/v1/router.py,sha256=RjuoWLpZer6Kl2BW_wznpNo6XL3BOpdTGqXZCn3QrrQ,1034
 nucliadb/writer/api/v1/services.py,sha256=3AUjk-SmvqJx76v7y89DZx6oyasojPliGYeniRQjpcU,13337
@@ -376,8 +376,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.7.0.post4734.dist-info/METADATA,sha256=LmyI9BUlxR1x6eJDoIY21ZN2TcyG-uVt7XNj3ZQt4eE,4158
-nucliadb-6.7.0.post4734.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb-6.7.0.post4734.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.7.0.post4734.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.7.0.post4734.dist-info/RECORD,,
+nucliadb-6.7.0.post4742.dist-info/METADATA,sha256=RMmnuJPlX0neyRjo1ojJkYdgNvjC6CTMQBcYFFidrgM,4158
+nucliadb-6.7.0.post4742.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb-6.7.0.post4742.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.7.0.post4742.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.7.0.post4742.dist-info/RECORD,,