nucliadb 6.9.1.post5180__py3-none-any.whl → 6.9.2.post5282__py3-none-any.whl

nucliadb/common/cluster/rollover.py

@@ -34,8 +34,10 @@ from nucliadb.common.external_index_providers.base import ExternalIndexManager
 from nucliadb.common.external_index_providers.manager import (
     get_external_index_manager,
 )
+from nucliadb.common.maindb.utils import get_driver
 from nucliadb.common.nidx import get_nidx_api_client
 from nucliadb.common.vector_index_config import nucliadb_index_config_to_nidx
+from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
 from nucliadb.migrator.settings import settings
 from nucliadb_protos import utils_pb2, writer_pb2
 from nucliadb_telemetry import errors
@@ -45,6 +47,7 @@ from .utils import (
     get_resource,
     get_rollover_resource_index_message,
     index_resource_to_shard,
+    wait_for_nidx,
 )
 
 logger = logging.getLogger(__name__)
@@ -254,6 +257,7 @@ async def index_to_rollover_index(
             for rid in resource_ids
         ]
         await asyncio.gather(*batch)
+        await wait_for_indexing_to_catch_up(app_context)
 
     async with datamanagers.with_transaction() as txn:
         state.resources_indexed = True
@@ -262,6 +266,22 @@ async def index_to_rollover_index(
         await txn.commit()
 
 
+async def wait_for_indexing_to_catch_up(app_context: ApplicationContext):
+    try:
+        app_context.nats_manager
+    except AssertionError:
+        logger.warning("Nats manager not initialized. Cannot wait for indexing to catch up")
+        return
+    max_pending = 1000
+    while True:
+        try:
+            await wait_for_nidx(app_context.nats_manager, max_wait_seconds=60, max_pending=max_pending)
+            return
+        except asyncio.TimeoutError:
+            logger.warning(f"Nidx is behind more than {max_pending} messages. Throttling rollover.")
+            await asyncio.sleep(30)
+
+
 async def _index_resource_to_rollover_index(
     app_context: ApplicationContext,
     rollover_shards: writer_pb2.Shards,
@@ -415,6 +435,15 @@ async def cutover_shards(app_context: ApplicationContext, kbid: str) -> None:
 
         await txn.commit()
 
+    # For KBs with pre-warm enabled, we must configure the new shards. There may
+    # be some small delay between this call and the shards being actually
+    # prewarmed, but rollovers are quite unusual and we prefer this rather than
+    # prewarming old and new shards at the same time
+    kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+    if kb_config is not None and kb_config.prewarm_enabled:
+        driver = get_driver()
+        await KnowledgeBox.configure_shards(driver, kbid, prewarm=True)
+
 
 async def validate_indexed_data(
     app_context: ApplicationContext, kbid: str, external: Optional[ExternalIndexManager] = None

nucliadb/common/cluster/utils.py

@@ -32,6 +32,7 @@ from nucliadb.common.cluster.settings import settings
 from nucliadb.ingest.orm import index_message
 from nucliadb.ingest.orm.resource import Resource
 from nucliadb_protos import writer_pb2
+from nucliadb_utils.nats import NatsConnectionManager
 from nucliadb_utils.utilities import Utility, clean_utility, get_utility, set_utility
 
 if TYPE_CHECKING:  # pragma: no cover
@@ -125,3 +126,28 @@ async def delete_resource_from_shard(
     partition = partitioning.generate_partition(kbid, resource_id)
 
     await sm.delete_resource(shard, resource_id, 0, str(partition), kbid)
+
+
+async def get_nats_consumer_pending_messages(
+    nats_manager: NatsConnectionManager, *, stream: str, consumer: str
+) -> int:
+    # get raw js client
+    js = nats_manager.js
+    consumer_info = await js.consumer_info(stream, consumer)
+    return consumer_info.num_pending
+
+
+async def wait_for_nidx(
+    nats_manager: NatsConnectionManager,
+    max_pending: int,
+    poll_interval_seconds: int = 5,
+    max_wait_seconds: int = 60,
+):
+    async with asyncio.timeout(max_wait_seconds):  # type: ignore
+        while True:
+            pending = await get_nats_consumer_pending_messages(
+                nats_manager, stream="nidx", consumer="nidx"
+            )
+            if pending < max_pending:
+                return
+            await asyncio.sleep(poll_interval_seconds)

nucliadb/common/datamanagers/atomic.py

@@ -42,6 +42,7 @@ from typing_extensions import Concatenate, ParamSpec
 
 from nucliadb.common.maindb.driver import Transaction
 
+from . import cluster as cluster_dm
 from . import kb as kb_dm
 from . import labels as labels_dm
 from . import resources as resources_dm
@@ -73,6 +74,10 @@ def rw_txn_wrap(fun: Callable[Concatenate[Transaction, P], Awaitable[T]]) -> Cal
     return wrapper
 
 
+class cluster:
+    get_kb_shards = ro_txn_wrap(cluster_dm.get_kb_shards)
+
+
 class kb:
     exists_kb = ro_txn_wrap(kb_dm.exists_kb)
     get_config = ro_txn_wrap(kb_dm.get_config)
@@ -83,6 +88,7 @@ class resources:
     get_resource_uuid_from_slug = ro_txn_wrap(resources_dm.get_resource_uuid_from_slug)
     resource_exists = ro_txn_wrap(resources_dm.resource_exists)
     slug_exists = ro_txn_wrap(resources_dm.slug_exists)
+    get_all_field_ids = ro_txn_wrap(resources_dm.get_all_field_ids)
 
 
 class labelset:

nucliadb/common/filter_expression.py

@@ -52,37 +52,20 @@ from nucliadb_models.filters import (
 )
 
 # Filters that end up as a facet
-FacetFilter = Union[
-    OriginTag,
-    Label,
-    ResourceMimetype,
-    FieldMimetype,
-    Entity,
-    Language,
-    OriginMetadata,
-    OriginPath,
-    Generated,
-    Kind,
-    OriginCollaborator,
-    OriginSource,
-    Status,
-]
-# In Python 3.9 we cannot do isinstance against an union
-# Once we support only 3.10+, we can remove this
-FacetFilterTypes = (
-    OriginTag,
-    Label,
-    ResourceMimetype,
-    FieldMimetype,
-    Entity,
-    Language,
-    OriginMetadata,
-    OriginPath,
-    Generated,
-    Kind,
-    OriginCollaborator,
-    OriginSource,
-    Status,
+FacetFilter = (
+    OriginTag
+    | Label
+    | ResourceMimetype
+    | FieldMimetype
+    | Entity
+    | Language
+    | OriginMetadata
+    | OriginPath
+    | Generated
+    | Kind
+    | OriginCollaborator
+    | OriginSource
+    | Status
 )
 
 
@@ -131,7 +114,7 @@ async def parse_expression(
             f.date.since.FromDatetime(expr.since)
         if expr.until:
             f.date.until.FromDatetime(expr.until)
-    elif isinstance(expr, FacetFilterTypes):
+    elif isinstance(expr, FacetFilter):
         f.facet.facet = facet_from_filter(expr)
     else:
         # This is a trick so mypy generates an error if this branch can be reached,

nucliadb/ingest/consumer/service.py

@@ -140,9 +140,8 @@ async def start_shard_creator() -> Callable[[], Awaitable[None]]:
     driver = await setup_driver()
     pubsub = await get_pubsub()
     assert pubsub is not None, "Pubsub is not configured"
-    storage = await get_storage(service_name=SERVICE_NAME)
 
-    shard_creator = ShardCreatorHandler(driver=driver, storage=storage, pubsub=pubsub)
+    shard_creator = ShardCreatorHandler(driver=driver, pubsub=pubsub)
     await shard_creator.initialize()
 
     return shard_creator.finalize

nucliadb/ingest/consumer/shard_creator.py

@@ -25,14 +25,14 @@ from typing import Any
 
 from nidx_protos import nodereader_pb2, noderesources_pb2
 
-from nucliadb.common import locking
+from nucliadb.common import datamanagers, locking
+from nucliadb.common.cluster.settings import settings
 from nucliadb.common.cluster.utils import get_shard_manager
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.common.nidx import get_nidx_api_client
 from nucliadb_protos import writer_pb2
 from nucliadb_utils import const
 from nucliadb_utils.cache.pubsub import PubSubDriver
-from nucliadb_utils.storages.storage import Storage
 
 from . import metrics
 from .utils import DelayedTaskHandler
@@ -52,12 +52,10 @@ class ShardCreatorHandler:
         self,
         *,
         driver: Driver,
-        storage: Storage,
         pubsub: PubSubDriver,
         check_delay: float = 10.0,
     ):
         self.driver = driver
-        self.storage = storage
         self.pubsub = pubsub
         self.shard_manager = get_shard_manager()
         self.task_handler = DelayedTaskHandler(check_delay)
@@ -111,4 +109,17 @@ class ShardCreatorHandler:
                     shard_id=noderesources_pb2.ShardId(id=current_shard.nidx_shard_id)
                 )  # type: ignore
             )
-            await self.shard_manager.maybe_create_new_shard(kbid, shard.paragraphs)
+
+            if not should_create_new_shard(shard.paragraphs):
+                return
+
+            logger.info({"message": "Adding shard", "kbid": kbid})
+            async with datamanagers.with_rw_transaction() as txn:
+                kb_config = await datamanagers.kb.get_config(txn, kbid=kbid)
+                prewarm = kb_config is not None and kb_config.prewarm_enabled
+                await self.shard_manager.create_shard_by_kbid(txn, kbid, prewarm_enabled=prewarm)
+                await txn.commit()
+
+
+def should_create_new_shard(num_paragraphs: int) -> bool:
+    return num_paragraphs > settings.max_shard_paragraphs

nucliadb/ingest/fields/base.py

@@ -47,10 +47,8 @@ from nucliadb_protos.resources_pb2 import (
 )
 from nucliadb_protos.utils_pb2 import ExtractedText, VectorObject
 from nucliadb_protos.writer_pb2 import Error, FieldStatus
-from nucliadb_utils import const
 from nucliadb_utils.storages.exceptions import CouldNotCopyNotFound
 from nucliadb_utils.storages.storage import Storage, StorageField
-from nucliadb_utils.utilities import has_feature
 
 logger = logging.getLogger(__name__)
 
@@ -224,21 +222,6 @@ class Field(Generic[PbType]):
     ) -> None:
         # Try delete vectors
         sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
-
-        if has_feature(const.Features.DEBUG_MISSING_VECTORS):
-            # This is a very chatty log. It is just a temporary hint while debugging an issue.
-            logger.info(
-                "Deleting vectors from storage",
-                extra={
-                    "kbid": self.kbid,
-                    "rid": self.resource.uuid,
-                    "field": f"{self.type}/{self.id}",
-                    "vectorset": vectorset,
-                    "storage_key_kind": storage_key_kind,
-                    "key": sf.key,
-                    "bucket": sf.bucket,
-                },
-            )
         try:
             await self.storage.delete_upload(sf.key, sf.bucket)
         except KeyError:

nucliadb/ingest/orm/knowledgebox.py

@@ -24,7 +24,7 @@ from uuid import uuid4
 
 from grpc import StatusCode
 from grpc.aio import AioRpcError
-from nidx_protos import noderesources_pb2
+from nidx_protos import nidx_pb2, noderesources_pb2
 
 from nucliadb.common import datamanagers
 from nucliadb.common.cluster.exceptions import ShardNotFound
@@ -108,6 +108,7 @@ class KnowledgeBox:
         external_index_provider: CreateExternalIndexProviderMetadata = CreateExternalIndexProviderMetadata(),
         hidden_resources_enabled: bool = False,
         hidden_resources_hide_on_creation: bool = False,
+        prewarm_enabled: bool = False,
     ) -> tuple[str, str]:
         """Creates a new knowledge box and return its id and slug."""
 
@@ -194,6 +195,7 @@ class KnowledgeBox:
                     migration_version=get_latest_version(),
                     hidden_resources_enabled=hidden_resources_enabled,
                     hidden_resources_hide_on_creation=hidden_resources_hide_on_creation,
+                    prewarm_enabled=prewarm_enabled,
                 )
                 config.external_index_provider.CopyFrom(stored_external_index_provider)
                 await datamanagers.kb.set_config(txn, kbid=kbid, config=config)
@@ -220,7 +222,7 @@ class KnowledgeBox:
                 shard_manager = get_shard_manager()
                 # XXX creating a shard is a slow IO operation that requires a write
                 # txn to be open!
-                await shard_manager.create_shard_by_kbid(txn, kbid)
+                await shard_manager.create_shard_by_kbid(txn, kbid, prewarm_enabled=prewarm_enabled)
                 # shards don't need a rollback as they will be eventually purged
 
                 await txn.commit()
@@ -243,39 +245,86 @@ class KnowledgeBox:
     @classmethod
     async def update(
         cls,
-        txn: Transaction,
-        uuid: str,
+        driver: Driver,
+        kbid: str,
+        *,
         slug: Optional[str] = None,
-        config: Optional[KnowledgeBoxConfig] = None,
+        title: Optional[str] = None,
+        description: Optional[str] = None,
+        migration_version: Optional[int] = None,
+        external_index_provider: Optional[StoredExternalIndexProviderMetadata] = None,
+        hidden_resources_enabled: Optional[bool] = None,
+        hidden_resources_hide_on_creation: Optional[bool] = None,
+        prewarm_enabled: Optional[bool] = None,
     ) -> str:
-        exist = await datamanagers.kb.get_config(txn, kbid=uuid, for_update=True)
-        if not exist:
-            raise datamanagers.exceptions.KnowledgeBoxNotFound()
-
-        if slug:
-            await txn.delete(datamanagers.kb.KB_SLUGS.format(slug=exist.slug))
-            await txn.set(
-                datamanagers.kb.KB_SLUGS.format(slug=slug),
-                uuid.encode(),
-            )
-            if config:
-                config.slug = slug
-            else:
-                exist.slug = slug
+        async with driver.rw_transaction() as txn:
+            stored = await datamanagers.kb.get_config(txn, kbid=kbid, for_update=True)
+            if not stored:
+                raise datamanagers.exceptions.KnowledgeBoxNotFound()
+
+            if slug:
+                await txn.delete(datamanagers.kb.KB_SLUGS.format(slug=stored.slug))
+                await txn.set(
+                    datamanagers.kb.KB_SLUGS.format(slug=slug),
+                    kbid.encode(),
+                )
+                stored.slug = slug
 
-        if config and exist != config:
-            exist.MergeFrom(config)
-            exist.hidden_resources_enabled = config.hidden_resources_enabled
-            exist.hidden_resources_hide_on_creation = config.hidden_resources_hide_on_creation
+            if title is not None:
+                stored.title = title
+            if description is not None:
+                stored.description = description
 
-        if exist.hidden_resources_hide_on_creation and not exist.hidden_resources_enabled:
-            raise KnowledgeBoxCreationError(
-                "Cannot hide new resources if the hidden resources feature is disabled"
-            )
+            if migration_version is not None:
+                stored.migration_version = migration_version
+
+            if external_index_provider is not None:
+                stored.external_index_provider.MergeFrom(external_index_provider)
 
-        await datamanagers.kb.set_config(txn, kbid=uuid, config=exist)
+            if hidden_resources_enabled is not None:
+                stored.hidden_resources_enabled = hidden_resources_enabled
+            if hidden_resources_hide_on_creation is not None:
+                stored.hidden_resources_hide_on_creation = hidden_resources_hide_on_creation
+
+            update_nidx_prewarm = None
+            if prewarm_enabled is not None:
+                if stored.prewarm_enabled != prewarm_enabled:
+                    update_nidx_prewarm = prewarm_enabled
+                stored.prewarm_enabled = prewarm_enabled
+
+            if stored.hidden_resources_hide_on_creation and not stored.hidden_resources_enabled:
+                raise KnowledgeBoxCreationError(
+                    "Cannot hide new resources if the hidden resources feature is disabled"
+                )
 
-        return uuid
+            await datamanagers.kb.set_config(txn, kbid=kbid, config=stored)
+
+            await txn.commit()
+
+        if update_nidx_prewarm is not None:
+            await cls.configure_shards(driver, kbid, prewarm=update_nidx_prewarm)
+
+        return kbid
+
+    @classmethod
+    async def configure_shards(cls, driver: Driver, kbid: str, *, prewarm: bool):
+        shards_obj = await datamanagers.atomic.cluster.get_kb_shards(kbid=kbid)
+        if shards_obj is None:
+            logger.warning(f"Shards not found for KB while updating pre-warm flag", extra={"kbid": kbid})
+            return
+
+        nidx_shard_ids = [shard.nidx_shard_id for shard in shards_obj.shards]
+
+        nidx_api = get_nidx_api_client()
+        if nidx_api is not None and len(nidx_shard_ids) > 0:
+            configs = [
+                nidx_pb2.ShardConfig(
+                    shard_id=shard_id,
+                    prewarm_enabled=prewarm,
+                )
+                for shard_id in nidx_shard_ids
+            ]
+            await nidx_api.ConfigureShards(nidx_pb2.ShardsConfig(configs=configs))
 
     @classmethod
     async def delete(cls, driver: Driver, kbid: str):

nucliadb/ingest/orm/processor/processor.py

@@ -446,22 +446,27 @@ class Processor:
             # a resource was move to another shard while it was being indexed
             shard_id = await datamanagers.resources.get_resource_shard_id(txn, kbid=kbid, rid=uuid)
 
-        shard = None
-        if shard_id is not None:
-            # Resource already has a shard assigned
-            shard = await kb.get_resource_shard(shard_id)
-            if shard is None:
-                raise AttributeError("Shard not available")
-        else:
-            # It's a new resource, get KB's current active shard to place new resource on
-            shard = await self.index_node_shard_manager.get_current_active_shard(txn, kbid)
-            if shard is None:
-                # No current shard available, create a new one
-                shard = await self.index_node_shard_manager.create_shard_by_kbid(txn, kbid)
-            await datamanagers.resources.set_resource_shard_id(
-                txn, kbid=kbid, rid=uuid, shard=shard.shard
-            )
-        return shard
+            shard = None
+            if shard_id is not None:
+                # Resource already has a shard assigned
+                shard = await kb.get_resource_shard(shard_id)
+                if shard is None:
+                    raise AttributeError("Shard not available")
+            else:
+                # It's a new resource, get KB's current active shard to place new resource on
+                shard = await self.index_node_shard_manager.get_current_active_shard(txn, kbid)
+                if shard is None:
+                    # No current shard available, create a new one
+                    async with locking.distributed_lock(locking.NEW_SHARD_LOCK.format(kbid=kbid)):
+                        kb_config = await datamanagers.kb.get_config(txn, kbid=kbid)
+                        prewarm = kb_config is not None and kb_config.prewarm_enabled
+                        shard = await self.index_node_shard_manager.create_shard_by_kbid(
+                            txn, kbid, prewarm_enabled=prewarm
+                        )
+                await datamanagers.resources.set_resource_shard_id(
+                    txn, kbid=kbid, rid=uuid, shard=shard.shard
+                )
+            return shard
 
     @processor_observer.wrap({"type": "index_resource"})
     async def index_resource(

nucliadb/ingest/service/writer.py

@@ -126,6 +126,7 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
                 external_index_provider=request.external_index_provider,
                 hidden_resources_enabled=request.hidden_resources_enabled,
                 hidden_resources_hide_on_creation=request.hidden_resources_hide_on_creation,
+                prewarm_enabled=request.prewarm_enabled,
             )
 
         except KnowledgeBoxConflict:
@@ -167,11 +168,17 @@ class WriterServicer(writer_pb2_grpc.WriterServicer):
             )
 
         try:
-            async with self.driver.rw_transaction() as txn:
-                kbid = await KnowledgeBoxORM.update(
-                    txn, uuid=request.uuid, slug=request.slug, config=request.config
-                )
-                await txn.commit()
+            kbid = await KnowledgeBoxORM.update(
+                self.driver,
+                kbid=request.uuid,
+                slug=request.slug,
+                title=request.config.title or None,
+                description=request.config.description or None,
+                external_index_provider=request.config.external_index_provider or None,
+                hidden_resources_enabled=request.config.hidden_resources_enabled,
+                hidden_resources_hide_on_creation=request.config.hidden_resources_hide_on_creation,
+                prewarm_enabled=request.config.prewarm_enabled,
+            )
         except KnowledgeBoxNotFound:
             return UpdateKnowledgeBoxResponse(status=KnowledgeBoxResponseStatus.NOTFOUND)
         except Exception:

nucliadb/migrator/datamanager.py

@@ -77,13 +77,7 @@ class MigrationsDataManager:
         return KnowledgeBoxInfo(current_version=kb_config.migration_version)
 
     async def update_kb_info(self, *, kbid: str, current_version: int) -> None:
-        async with self.driver.rw_transaction() as txn:
-            kb_config = await datamanagers.kb.get_config(txn, kbid=kbid, for_update=True)
-            if kb_config is None:
-                raise Exception(f"KB {kbid} does not exist")
-            kb_config.migration_version = current_version
-            await KnowledgeBoxORM.update(txn, kbid, config=kb_config)
-            await txn.commit()
+        await KnowledgeBoxORM.update(self.driver, kbid, migration_version=current_version)
 
     async def get_global_info(self) -> GlobalInfo:
         async with self.driver.ro_transaction() as txn:

nucliadb/purge/__init__.py

@@ -19,6 +19,7 @@
 #
 import asyncio
 import importlib.metadata
+from itertools import batched  # type: ignore
 from typing import AsyncGenerator
 
 from nucliadb.common import datamanagers
@@ -233,7 +234,7 @@ async def purge_kb_vectorsets(driver: Driver, storage: Storage):
                     fields.extend((await resource.get_fields(force=True)).values())
 
             logger.info(f"Purging {len(fields)} fields for vectorset {vectorset}", extra={"kbid": kbid})
-            for fields_batch in batchify(fields, 20):
+            for fields_batch in batched(fields, n=20):
                 tasks = []
                 for field in fields_batch:
                     if purge_payload.storage_key_kind == VectorSetConfig.StorageKeyKind.UNSET:
@@ -317,9 +318,3 @@ def run() -> int:  # pragma: no cover
     setup_logging()
     errors.setup_error_handling(importlib.metadata.distribution("nucliadb").version)
     return asyncio.run(main())
-
-
-def batchify(iterable, n=1):
-    """Yield successive n-sized chunks from iterable."""
-    for i in range(0, len(iterable), n):
-        yield iterable[i : i + n]

nucliadb/reader/api/v1/learning_config.py

@@ -128,6 +128,27 @@ async def get_schema_for_configuration_updates(
     )
 
 
+@api.get(
+    path=f"/{KB_PREFIX}/{{kbid}}/generative_providers",
+    status_code=200,
+    summary="Available models for a knowledge box",
+    description="Get all available models for a knowledge box grouped by provider",
+    response_model=None,
+    tags=["Models"],
+)
+@requires_one([NucliaDBRoles.READER, NucliaDBRoles.MANAGER])
+@version(1)
+async def get_models_group_by_providers(
+    request: Request, kbid: str, x_nucliadb_account: str = Header(default="", include_in_schema=False)
+):
+    return await learning_config_proxy(
+        request,
+        "GET",
+        f"/generative_providers/{kbid}",
+        headers={"account-id": x_nucliadb_account},
+    )
+
+
 @api.get(
     path=f"/nua/schema",
     status_code=200,

nucliadb/search/api/v1/find.py

@@ -46,7 +46,6 @@ from nucliadb_models.search import (
     KnowledgeboxFindResults,
     NucliaDBClientType,
     RankFusionName,
-    Reranker,
     RerankerName,
     ResourceProperties,
     SearchParamDefaults,
@@ -127,11 +126,10 @@ async def find_knowledgebox(
     extracted: list[ExtractedDataTypeName] = fastapi_query(SearchParamDefaults.extracted),
     with_duplicates: bool = fastapi_query(SearchParamDefaults.with_duplicates),
     with_synonyms: bool = fastapi_query(SearchParamDefaults.with_synonyms),
-    autofilter: bool = fastapi_query(SearchParamDefaults.autofilter),
     security_groups: list[str] = fastapi_query(SearchParamDefaults.security_groups),
     show_hidden: bool = fastapi_query(SearchParamDefaults.show_hidden),
     rank_fusion: RankFusionName = fastapi_query(SearchParamDefaults.rank_fusion),
-    reranker: Union[RerankerName, Reranker] = fastapi_query(SearchParamDefaults.reranker),
+    reranker: RerankerName = fastapi_query(SearchParamDefaults.reranker),
     search_configuration: Optional[str] = Query(
         default=None,
         description="Load find parameters from this configuration. Parameters in the request override parameters from the configuration.",
@@ -166,7 +164,6 @@ async def find_knowledgebox(
             extracted=extracted,
             with_duplicates=with_duplicates,
             with_synonyms=with_synonyms,
-            autofilter=autofilter,
             security=security,
             show_hidden=show_hidden,
             rank_fusion=rank_fusion,

nucliadb/search/api/v1/resource/ask.py

@@ -28,7 +28,8 @@ from nucliadb.search.api.v1.resource.utils import get_resource_uuid_by_slug
 from nucliadb.search.api.v1.router import KB_PREFIX, RESOURCE_SLUG_PREFIX, api
 from nucliadb_models.resource import NucliaDBRoles
 from nucliadb_models.search import AskRequest, NucliaDBClientType, SyncAskResponse
-from nucliadb_utils.authentication import requires
+from nucliadb_models.security import RequestSecurity
+from nucliadb_utils.authentication import NucliaUser, requires
 
 from ..ask import create_ask_response
 
@@ -58,6 +59,15 @@ async def resource_ask_endpoint_by_uuid(
         "This is slower and requires waiting for entire answer to be ready.",
     ),
 ) -> Union[StreamingResponse, HTTPClientError, Response]:
+    current_user: NucliaUser = request.user
+    # If present, security groups from AuthorizationBackend overrides any
+    # security group of the payload
+    if current_user.security_groups:
+        if item.security is None:
+            item.security = RequestSecurity(groups=current_user.security_groups)
+        else:
+            item.security.groups = current_user.security_groups
+
     return await create_ask_response(
         kbid=kbid,
         ask_request=item,
@@ -98,6 +108,16 @@ async def resource_ask_endpoint_by_slug(
     resource_id = await get_resource_uuid_by_slug(kbid, slug)
     if resource_id is None:
         return HTTPClientError(status_code=404, detail="Resource not found")
+
+    current_user: NucliaUser = request.user
+    # If present, security groups from AuthorizationBackend overrides any
+    # security group of the payload
+    if current_user.security_groups:
+        if item.security is None:
+            item.security = RequestSecurity(groups=current_user.security_groups)
+        else:
+            item.security.groups = current_user.security_groups
+
     return await create_ask_response(
         kbid=kbid,
         ask_request=item,