nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2798__py3-none-any.whl

nucliadb/search/api/v1/catalog.py

@@ -0,0 +1,184 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+from time import time
+from typing import Optional, Union
+
+from fastapi import Request, Response
+from fastapi_versioning import version
+
+from nucliadb.common.datamanagers.exceptions import KnowledgeBoxNotFound
+from nucliadb.common.maindb.pg import PGDriver
+from nucliadb.common.maindb.utils import get_driver
+from nucliadb.models.responses import HTTPClientError
+from nucliadb.search import logger
+from nucliadb.search.api.v1.router import KB_PREFIX, api
+from nucliadb.search.api.v1.utils import fastapi_query
+from nucliadb.search.search import cache
+from nucliadb.search.search.exceptions import InvalidQueryError
+from nucliadb.search.search.merge import fetch_resources
+from nucliadb.search.search.pgcatalog import pgcatalog_search
+from nucliadb.search.search.query_parser.parser import parse_catalog
+from nucliadb.search.search.utils import (
+    maybe_log_request_payload,
+)
+from nucliadb_models.common import FieldTypeName
+from nucliadb_models.metadata import ResourceProcessingStatus
+from nucliadb_models.resource import NucliaDBRoles
+from nucliadb_models.search import (
+    CatalogRequest,
+    CatalogResponse,
+    KnowledgeboxSearchResults,
+    ResourceProperties,
+    SearchParamDefaults,
+    SortField,
+    SortOptions,
+    SortOrder,
+)
+from nucliadb_models.utils import DateTime
+from nucliadb_utils.authentication import requires
+from nucliadb_utils.exceptions import LimitsExceededError
+
+
+@api.get(
+    f"/{KB_PREFIX}/{{kbid}}/catalog",
+    status_code=200,
+    summary="List resources of a Knowledge Box",
+    description="List resources of a Knowledge Box",
+    response_model=KnowledgeboxSearchResults,
+    response_model_exclude_unset=True,
+    tags=["Search"],
+)
+@requires(NucliaDBRoles.READER)
+@version(1)
+async def catalog_get(
+    request: Request,
+    response: Response,
+    kbid: str,
+    query: str = fastapi_query(SearchParamDefaults.query),
+    filters: list[str] = fastapi_query(SearchParamDefaults.filters),
+    faceted: list[str] = fastapi_query(SearchParamDefaults.faceted),
+    sort_field: SortField = fastapi_query(SearchParamDefaults.sort_field),
+    sort_limit: Optional[int] = fastapi_query(SearchParamDefaults.sort_limit),
+    sort_order: SortOrder = fastapi_query(SearchParamDefaults.sort_order),
+    page_number: int = fastapi_query(SearchParamDefaults.catalog_page_number),
+    page_size: int = fastapi_query(SearchParamDefaults.catalog_page_size),
+    shards: list[str] = fastapi_query(SearchParamDefaults.shards, deprecated=True),
+    with_status: Optional[ResourceProcessingStatus] = fastapi_query(
+        SearchParamDefaults.with_status, deprecated="Use filters instead"
+    ),
+    debug: bool = fastapi_query(SearchParamDefaults.debug, include_in_schema=False),
+    range_creation_start: Optional[DateTime] = fastapi_query(SearchParamDefaults.range_creation_start),
+    range_creation_end: Optional[DateTime] = fastapi_query(SearchParamDefaults.range_creation_end),
+    range_modification_start: Optional[DateTime] = fastapi_query(
+        SearchParamDefaults.range_modification_start
+    ),
+    range_modification_end: Optional[DateTime] = fastapi_query(
+        SearchParamDefaults.range_modification_end
+    ),
+    hidden: Optional[bool] = fastapi_query(SearchParamDefaults.hidden),
+) -> Union[KnowledgeboxSearchResults, HTTPClientError]:
+    item = CatalogRequest(
+        query=query,
+        filters=filters,
+        faceted=faceted,
+        page_number=page_number,
+        page_size=page_size,
+        shards=shards,
+        debug=debug,
+        with_status=with_status,
+        range_creation_start=range_creation_start,
+        range_creation_end=range_creation_end,
+        range_modification_start=range_modification_start,
+        range_modification_end=range_modification_end,
+        hidden=hidden,
+    )
+    if sort_field:
+        item.sort = SortOptions(field=sort_field, limit=sort_limit, order=sort_order)
+    return await catalog(kbid, item)
+
+
+@api.post(
+    f"/{KB_PREFIX}/{{kbid}}/catalog",
+    status_code=200,
+    summary="List resources of a Knowledge Box",
+    description="List resources of a Knowledge Box",
+    response_model=KnowledgeboxSearchResults,
+    response_model_exclude_unset=True,
+    tags=["Search"],
+)
+@requires(NucliaDBRoles.READER)
+@version(1)
+async def catalog_post(
+    request: Request,
+    kbid: str,
+    item: CatalogRequest,
+) -> Union[CatalogResponse, HTTPClientError]:
+    return await catalog(kbid, item)
+
+
+async def catalog(
+    kbid: str,
+    item: CatalogRequest,
+):
+    """
+    Catalog endpoint is a simplified version of the search endpoint, it only
+    returns bm25 results on titles and it does not support vector search.
+    It is useful for listing resources in a knowledge box.
+    """
+    if not pgcatalog_enabled():  # pragma: no cover
+        return HTTPClientError(status_code=501, detail="PG driver is needed for catalog search")
+
+    maybe_log_request_payload(kbid, "/catalog", item)
+    start_time = time()
+    try:
+        with cache.request_caches():
+            query_parser = parse_catalog(kbid, item)
+
+            catalog_results = CatalogResponse()
+            catalog_results.fulltext = await pgcatalog_search(query_parser)
+            catalog_results.resources = await fetch_resources(
+                resources=[r.rid for r in catalog_results.fulltext.results],
+                kbid=kbid,
+                show=[ResourceProperties.BASIC, ResourceProperties.ERRORS],
+                field_type_filter=list(FieldTypeName),
+                extracted=[],
+            )
+            return catalog_results
+    except InvalidQueryError as exc:
+        return HTTPClientError(status_code=412, detail=str(exc))
+    except KnowledgeBoxNotFound:
+        return HTTPClientError(status_code=404, detail="Knowledge Box not found")
+    except LimitsExceededError as exc:
+        return HTTPClientError(status_code=exc.status_code, detail=exc.detail)
+    finally:
+        duration = time() - start_time
+        if duration > 2:  # pragma: no cover
+            logger.warning(
+                "Slow catalog request",
+                extra={
+                    "kbid": kbid,
+                    "duration": duration,
+                    "query": item.model_dump_json(),
+                },
+            )
+
+
+def pgcatalog_enabled():
+    return isinstance(get_driver(), PGDriver)

nucliadb/search/api/v1/feedback.py

@@ -18,18 +18,18 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 
-
 from fastapi import Header, Request, Response
 from fastapi_versioning import version
 
+from nucliadb.common.models_utils import to_proto
 from nucliadb.models.responses import HTTPClientError
-from nucliadb.search import logger, predict
+from nucliadb.search import logger
 from nucliadb.search.api.v1.router import KB_PREFIX, api
-from nucliadb.search.utilities import get_predict
 from nucliadb_models.resource import NucliaDBRoles
 from nucliadb_models.search import FeedbackRequest, NucliaDBClientType
 from nucliadb_telemetry import errors
 from nucliadb_utils.authentication import requires
+from nucliadb_utils.utilities import get_audit
 
 
 @api.post(
@@ -51,28 +51,20 @@ async def send_feedback_endpoint(
     x_forwarded_for: str = Header(""),
 ):
     try:
-        return await send_feedback(
-            kbid, item, x_nucliadb_user, x_ndb_client, x_forwarded_for
-        )
-    except predict.ProxiedPredictAPIError as err:
-        return HTTPClientError(
-            status_code=err.status,
-            detail=err.detail,
-        )
+        audit = get_audit()
+        if audit is not None:
+            audit.feedback(
+                kbid=kbid,
+                user=x_nucliadb_user,
+                client_type=to_proto.client_type(x_ndb_client),
+                origin=x_forwarded_for,
+                learning_id=item.ident,
+                good=item.good,
+                task=to_proto.feedback_task(item.task),
+                feedback=item.feedback,
+                text_block_id=item.text_block_id,
+            )
     except Exception as ex:
         errors.capture_exception(ex)
         logger.exception("Unexpected error sending feedback", extra={"kbid": kbid})
         return HTTPClientError(status_code=500, detail=f"Internal server error")
-
-
-async def send_feedback(
-    kbid: str,
-    item: FeedbackRequest,
-    x_nucliadb_user: str,
-    x_ndb_client: NucliaDBClientType,
-    x_forwarded_for: str,
-):
-    predict = get_predict()
-    await predict.send_feedback(
-        kbid, item, x_nucliadb_user, x_ndb_client, x_forwarded_for
-    )

nucliadb/search/api/v1/find.py

@@ -18,7 +18,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import json
-from datetime import datetime
 from typing import Optional, Union
 
 from fastapi import Body, Header, Query, Request, Response
@@ -31,20 +30,25 @@ from nucliadb.models.responses import HTTPClientError
 from nucliadb.search import predict
 from nucliadb.search.api.v1.router import KB_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
+from nucliadb.search.search import cache
 from nucliadb.search.search.exceptions import InvalidQueryError
 from nucliadb.search.search.find import find
-from nucliadb.search.search.utils import min_score_from_query_params
+from nucliadb.search.search.utils import maybe_log_request_payload, min_score_from_query_params
 from nucliadb_models.common import FieldTypeName
 from nucliadb_models.resource import ExtractedDataTypeName, NucliaDBRoles
 from nucliadb_models.search import (
     FindRequest,
     KnowledgeboxFindResults,
     NucliaDBClientType,
+    RankFusionName,
+    Reranker,
+    RerankerName,
     ResourceProperties,
     SearchOptions,
     SearchParamDefaults,
 )
 from nucliadb_models.security import RequestSecurity
+from nucliadb_models.utils import DateTime
 from nucliadb_utils.authentication import requires
 from nucliadb_utils.exceptions import LimitsExceededError
 
@@ -54,7 +58,7 @@ FIND_EXAMPLES = {
         description="Perform a hybrid search that will return text and semantic results matching the query",
         value={
             "query": "How can I be an effective product manager?",
-            "features": [SearchOptions.PARAGRAPH, SearchOptions.VECTOR],
+            "features": [SearchOptions.KEYWORD, SearchOptions.SEMANTIC],
         },
     )
 }
@@ -78,39 +82,35 @@ async def find_knowledgebox(
     query: str = fastapi_query(SearchParamDefaults.query),
     fields: list[str] = fastapi_query(SearchParamDefaults.fields),
     filters: list[str] = fastapi_query(SearchParamDefaults.filters),
-    page_number: int = fastapi_query(SearchParamDefaults.page_number),
-    page_size: int = fastapi_query(SearchParamDefaults.page_size),
+    top_k: Optional[int] = fastapi_query(SearchParamDefaults.top_k),
     min_score: Optional[float] = Query(
         default=None,
-        description="Minimum similarity score to filter vector index results. If not specified, the default minimum score of the semantic model associated to the Knowledge Box will be used. Check out the documentation for more information on how to use this parameter: https://docs.nuclia.dev/docs/docs/using/search/#minimum-score",  # noqa: E501
+        description="Minimum similarity score to filter vector index results. If not specified, the default minimum score of the semantic model associated to the Knowledge Box will be used. Check out the documentation for more information on how to use this parameter: https://docs.nuclia.dev/docs/rag/advanced/search#minimum-score",  # noqa: E501
         deprecated=True,
     ),
     min_score_semantic: Optional[float] = Query(
         default=None,
-        description="Minimum semantic similarity score to filter vector index results. If not specified, the default minimum score of the semantic model associated to the Knowledge Box will be used. Check out the documentation for more information on how to use this parameter: https://docs.nuclia.dev/docs/docs/using/search/#minimum-score",  # noqa: E501
+        description="Minimum semantic similarity score to filter vector index results. If not specified, the default minimum score of the semantic model associated to the Knowledge Box will be used. Check out the documentation for more information on how to use this parameter: https://docs.nuclia.dev/docs/rag/advanced/search#minimum-score",  # noqa: E501
     ),
     min_score_bm25: float = Query(
         default=0,
         description="Minimum bm25 score to filter paragraph and document index results",
         ge=0,
     ),
-    range_creation_start: Optional[datetime] = fastapi_query(
-        SearchParamDefaults.range_creation_start
-    ),
-    range_creation_end: Optional[datetime] = fastapi_query(
-        SearchParamDefaults.range_creation_end
-    ),
-    range_modification_start: Optional[datetime] = fastapi_query(
+    vectorset: Optional[str] = fastapi_query(SearchParamDefaults.vectorset),
+    range_creation_start: Optional[DateTime] = fastapi_query(SearchParamDefaults.range_creation_start),
+    range_creation_end: Optional[DateTime] = fastapi_query(SearchParamDefaults.range_creation_end),
+    range_modification_start: Optional[DateTime] = fastapi_query(
         SearchParamDefaults.range_modification_start
     ),
-    range_modification_end: Optional[datetime] = fastapi_query(
+    range_modification_end: Optional[DateTime] = fastapi_query(
         SearchParamDefaults.range_modification_end
     ),
     features: list[SearchOptions] = fastapi_query(
         SearchParamDefaults.search_features,
         default=[
-            SearchOptions.PARAGRAPH,
-            SearchOptions.VECTOR,
+            SearchOptions.KEYWORD,
+            SearchOptions.SEMANTIC,
         ],
     ),
     debug: bool = fastapi_query(SearchParamDefaults.debug),
@@ -119,13 +119,14 @@ async def find_knowledgebox(
     field_type_filter: list[FieldTypeName] = fastapi_query(
         SearchParamDefaults.field_type_filter, alias="field_type"
     ),
-    extracted: list[ExtractedDataTypeName] = fastapi_query(
-        SearchParamDefaults.extracted
-    ),
+    extracted: list[ExtractedDataTypeName] = fastapi_query(SearchParamDefaults.extracted),
     with_duplicates: bool = fastapi_query(SearchParamDefaults.with_duplicates),
     with_synonyms: bool = fastapi_query(SearchParamDefaults.with_synonyms),
     autofilter: bool = fastapi_query(SearchParamDefaults.autofilter),
     security_groups: list[str] = fastapi_query(SearchParamDefaults.security_groups),
+    show_hidden: bool = fastapi_query(SearchParamDefaults.show_hidden),
+    rank_fusion: RankFusionName = fastapi_query(SearchParamDefaults.rank_fusion),
+    reranker: Union[RerankerName, Reranker] = fastapi_query(SearchParamDefaults.reranker),
     x_ndb_client: NucliaDBClientType = Header(NucliaDBClientType.API),
     x_nucliadb_user: str = Header(""),
     x_forwarded_for: str = Header(""),
@@ -138,11 +139,9 @@ async def find_knowledgebox(
             query=query,
             fields=fields,
             filters=filters,
-            page_number=page_number,
-            page_size=page_size,
-            min_score=min_score_from_query_params(
-                min_score_bm25, min_score_semantic, min_score
-            ),
+            top_k=top_k,  # type: ignore
+            min_score=min_score_from_query_params(min_score_bm25, min_score_semantic, min_score),
+            vectorset=vectorset,
             range_creation_end=range_creation_end,
             range_creation_start=range_creation_start,
             range_modification_end=range_modification_end,
@@ -157,14 +156,15 @@ async def find_knowledgebox(
             with_synonyms=with_synonyms,
             autofilter=autofilter,
             security=security,
+            show_hidden=show_hidden,
+            rank_fusion=rank_fusion,
+            reranker=reranker,
         )
     except ValidationError as exc:
         detail = json.loads(exc.json())
         return HTTPClientError(status_code=422, detail=detail)
 
-    return await _find_endpoint(
-        response, kbid, item, x_ndb_client, x_nucliadb_user, x_forwarded_for
-    )
+    return await _find_endpoint(response, kbid, item, x_ndb_client, x_nucliadb_user, x_forwarded_for)
 
 
 @api.post(
@@ -187,9 +187,7 @@ async def find_post_knowledgebox(
     x_nucliadb_user: str = Header(""),
     x_forwarded_for: str = Header(""),
 ) -> Union[KnowledgeboxFindResults, HTTPClientError]:
-    return await _find_endpoint(
-        response, kbid, item, x_ndb_client, x_nucliadb_user, x_forwarded_for
-    )
+    return await _find_endpoint(response, kbid, item, x_ndb_client, x_nucliadb_user, x_forwarded_for)
 
 
 async def _find_endpoint(
@@ -201,11 +199,13 @@ async def _find_endpoint(
     x_forwarded_for: str,
 ) -> Union[KnowledgeboxFindResults, HTTPClientError]:
     try:
-        results, incomplete, _ = await find(
-            kbid, item, x_ndb_client, x_nucliadb_user, x_forwarded_for
-        )
-        response.status_code = 206 if incomplete else 200
-        return results
+        maybe_log_request_payload(kbid, "/find", item)
+        with cache.request_caches():
+            results, incomplete, _ = await find(
+                kbid, item, x_ndb_client, x_nucliadb_user, x_forwarded_for
+            )
+            response.status_code = 206 if incomplete else 200
+            return results
     except KnowledgeBoxNotFound:
         return HTTPClientError(status_code=404, detail="Knowledge Box not found")
     except LimitsExceededError as exc:

nucliadb/search/api/v1/knowledgebox.py

@@ -24,29 +24,35 @@ from fastapi import HTTPException, Request
 from fastapi_versioning import version
 from grpc import StatusCode as GrpcStatusCode
 from grpc.aio import AioRpcError
-from nucliadb_protos.noderesources_pb2 import Shard
-from nucliadb_protos.writer_pb2 import ShardObject as PBShardObject
-from nucliadb_protos.writer_pb2 import Shards
 
 from nucliadb.common import datamanagers
 from nucliadb.common.cluster.exceptions import ShardsNotFound
 from nucliadb.common.cluster.manager import choose_node
 from nucliadb.common.cluster.utils import get_shard_manager
+from nucliadb.common.constants import AVG_PARAGRAPH_SIZE_BYTES
+from nucliadb.common.counters import IndexCounts
+from nucliadb.common.external_index_providers.manager import get_external_index_manager
+from nucliadb.common.models_utils import from_proto
 from nucliadb.search import logger
 from nucliadb.search.api.v1.router import KB_PREFIX, api
 from nucliadb.search.api.v1.utils import fastapi_query
 from nucliadb.search.search.shards import get_shard
 from nucliadb.search.settings import settings
+from nucliadb_models.internal.shards import KnowledgeboxShards
 from nucliadb_models.resource import NucliaDBRoles
 from nucliadb_models.search import (
     KnowledgeboxCounters,
-    KnowledgeboxShards,
     SearchParamDefaults,
 )
+from nucliadb_protos.noderesources_pb2 import Shard
+from nucliadb_protos.writer_pb2 import ShardObject as PBShardObject
+from nucliadb_protos.writer_pb2 import Shards
 from nucliadb_telemetry import errors
+from nucliadb_utils import const
 from nucliadb_utils.authentication import requires, requires_one
+from nucliadb_utils.utilities import has_feature
 
-AVG_PARAGRAPH_SIZE_BYTES = 10_000
+MAX_PARAGRAPHS_FOR_SMALL_KB = 250_000
 
 
 @api.get(
@@ -68,7 +74,7 @@ async def knowledgebox_shards(request: Request, kbid: str) -> KnowledgeboxShards
             status_code=404,
             detail="The knowledgebox or its shards configuration is missing",
         )
-    return KnowledgeboxShards.from_message(shards)
+    return from_proto.kb_shards(shards)
 
 
 @api.get(
@@ -86,21 +92,83 @@ async def knowledgebox_counters(
     kbid: str,
     debug: bool = fastapi_query(SearchParamDefaults.debug),
 ) -> KnowledgeboxCounters:
-    shard_manager = get_shard_manager()
-
     try:
-        shard_groups: list[PBShardObject] = await shard_manager.get_shards_by_kbid(kbid)
+        return await _kb_counters(kbid, debug=debug)
     except ShardsNotFound:
         raise HTTPException(
             status_code=404,
             detail="The knowledgebox or its shards configuration is missing",
         )
 
+
+async def _kb_counters(
+    kbid: str,
+    debug: bool = False,
+) -> KnowledgeboxCounters:
+    """
+    Resources count is calculated from maindb and cached
+    Field count is calculated from the index node cluster
+    Paragraphs and Sentences count is calculated from the index node cluster or the external index provider.
+    Index size is estimated from the paragraphs count.
+    """
+    counters = KnowledgeboxCounters(
+        resources=0,
+        paragraphs=0,
+        fields=0,
+        sentences=0,
+        index_size=0,
+    )
+    external_index_manager = await get_external_index_manager(kbid)
+    if external_index_manager is not None:
+        index_counts = await external_index_manager.get_index_counts()
+        counters.paragraphs = index_counts.paragraphs
+        counters.sentences = index_counts.sentences
+        is_small_kb = index_counts.paragraphs < MAX_PARAGRAPHS_FOR_SMALL_KB
+        resource_count = await get_resources_count(kbid, force_calculate=is_small_kb)
+        # TODO: Find a way to query the fields count from the external index provider or use the catalog
+        counters.resources = counters.fields = resource_count
+    else:
+        node_index_counts, queried_shards = await get_node_index_counts(kbid)
+        counters.fields = node_index_counts.fields
+        counters.paragraphs = node_index_counts.paragraphs
+        counters.sentences = node_index_counts.sentences
+        is_small_kb = node_index_counts.paragraphs < MAX_PARAGRAPHS_FOR_SMALL_KB
+        resource_count = await get_resources_count(kbid, force_calculate=is_small_kb)
+        counters.resources = resource_count
+    counters.index_size = counters.paragraphs * AVG_PARAGRAPH_SIZE_BYTES
+    if debug and queried_shards is not None:
+        counters.shards = queried_shards
+    return counters
+
+
+async def get_resources_count(kbid: str, force_calculate: bool = False) -> int:
+    async with datamanagers.with_ro_transaction() as txn:
+        if force_calculate:
+            # For small kbs, this is faster and more up to date
+            resource_count = await datamanagers.resources.calculate_number_of_resources(txn, kbid=kbid)
+        else:
+            resource_count = await datamanagers.resources.get_number_of_resources(txn, kbid=kbid)
+            if resource_count == -1:
+                # WARNING: standalone, this value will never be cached
+                resource_count = await datamanagers.resources.calculate_number_of_resources(
+                    txn, kbid=kbid
+                )
+    return resource_count
+
+
+async def get_node_index_counts(kbid: str) -> tuple[IndexCounts, list[str]]:
+    """
+    Get the index counts for a knowledgebox that has an index in the index node cluster.
+    """
+    shard_manager = get_shard_manager()
+    shard_groups: list[PBShardObject] = await shard_manager.get_shards_by_kbid(kbid)
     ops = []
     queried_shards = []
     for shard_object in shard_groups:
         try:
-            node, shard_id = choose_node(shard_object)
+            node, shard_id = choose_node(
+                shard_object, use_nidx=has_feature(const.Features.NIDX_READS, context={"kbid": kbid})
+            )
         except KeyError:
             raise HTTPException(
                 status_code=500,
@@ -121,7 +189,7 @@ async def knowledgebox_counters(
         )
 
     try:
-        results: Optional[list[Shard]] = await asyncio.wait_for(  # type: ignore
+        results: Optional[list[Shard]] = await asyncio.wait_for(
             asyncio.gather(*ops, return_exceptions=True),  # type: ignore
             timeout=settings.search_timeout,
         )
@@ -138,56 +206,17 @@ async def knowledgebox_counters(
     if results is None:
         raise HTTPException(status_code=503, detail=f"No shards found")
 
-    field_count = 0
-    paragraph_count = 0
-    sentence_count = 0
-
+    counts = IndexCounts(
+        fields=0,
+        paragraphs=0,
+        sentences=0,
+    )
     for shard in results:
         if isinstance(shard, Exception):
             logger.error("Error getting shard info", exc_info=shard)
             errors.capture_exception(shard)
-            raise HTTPException(
-                status_code=500, detail=f"Error while geting shard data"
-            )
-
-        field_count += shard.fields
-        paragraph_count += shard.paragraphs
-        sentence_count += shard.sentences
-
-    async with datamanagers.with_transaction() as txn:
-        try:
-            if len(shard_groups) <= 1:
-                # for smaller kbs, this is faster and more up to date
-                resource_count = (
-                    await datamanagers.resources.calculate_number_of_resources(
-                        txn, kbid=kbid
-                    )
-                )
-            else:
-                resource_count = await datamanagers.resources.get_number_of_resources(
-                    txn, kbid=kbid
-                )
-                if resource_count == -1:
-                    # WARNING: standalone, this value will never be cached
-                    resource_count = (
-                        await datamanagers.resources.calculate_number_of_resources(
-                            txn, kbid=kbid
-                        )
-                    )
-        except Exception as exc:
-            errors.capture_exception(exc)
-            raise HTTPException(
-                status_code=500, detail="Couldn't retrieve counters right now"
-            )
-
-    counters = KnowledgeboxCounters(
-        resources=resource_count,
-        paragraphs=paragraph_count,
-        fields=field_count,
-        sentences=sentence_count,
-        index_size=paragraph_count * AVG_PARAGRAPH_SIZE_BYTES,
-    )
-
-    if debug:
-        counters.shards = queried_shards
-    return counters
+            raise HTTPException(status_code=500, detail=f"Error while geting shard data")
+        counts.fields += shard.fields
+        counts.paragraphs += shard.paragraphs
+        counts.sentences += shard.sentences
+    return counts, queried_shards

nucliadb/search/api/v1/resource/ask.py

@@ -40,8 +40,6 @@ from ..ask import create_ask_response
     description="Ask questions to a resource",
     tags=["Search"],
     response_model=SyncAskResponse,
-    # Add this to OpenAPI schema when endpoint is not in beta anymore
-    include_in_schema=False,
 )
 @requires(NucliaDBRoles.READER)
 @version(1)
@@ -77,8 +75,6 @@ async def resource_ask_endpoint_by_uuid(
     description="Ask questions to a resource",
     tags=["Search"],
     response_model=SyncAskResponse,
-    # Add this to OpenAPI schema when endpoint is not in beta anymore
-    include_in_schema=False,
 )
 @requires(NucliaDBRoles.READER)
 @version(1)
@@ -111,7 +107,5 @@ async def resource_ask_endpoint_by_slug(
 
 
 async def get_resource_uuid_by_slug(kbid: str, slug: str) -> Optional[str]:
-    async with datamanagers.with_transaction() as txn:
-        return await datamanagers.resources.get_resource_uuid_from_slug(
-            txn, kbid=kbid, slug=slug
-        )
+    async with datamanagers.with_ro_transaction() as txn:
+        return await datamanagers.resources.get_resource_uuid_from_slug(txn, kbid=kbid, slug=slug)