nucliadb 2.46.1.post382__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/writer/api/v1/resource.py

@@ -25,17 +25,6 @@ from uuid import uuid4
 
 from fastapi import HTTPException, Query, Response
 from fastapi_versioning import version
-from grpc import StatusCode as GrpcStatusCode
-from grpc.aio import AioRpcError
-from nucliadb_protos.resources_pb2 import Metadata
-from nucliadb_protos.writer_pb2 import (
-    BrokerMessage,
-    IndexResource,
-    ResourceFieldExistsResponse,
-    ResourceFieldId,
-    ResourceIdRequest,
-    ResourceIdResponse,
-)
 from starlette.requests import Request
 
 from nucliadb.common import datamanagers
@@ -46,7 +35,8 @@ from nucliadb.common.maindb.utils import get_driver
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
 from nucliadb.ingest.processing import ProcessingInfo, PushPayload, Source
 from nucliadb.writer import SERVICE_NAME, logger
-from nucliadb.writer.api.constants import SKIP_STORE_DEFAULT, SYNC_CALL, X_NUCLIADB_USER
+from nucliadb.writer.api.constants import SKIP_STORE_DEFAULT, X_NUCLIADB_USER
+from nucliadb.writer.api.v1 import transaction
 from nucliadb.writer.api.v1.router import (
     KB_PREFIX,
     RESOURCE_PREFIX,
@@ -54,24 +44,17 @@ from nucliadb.writer.api.v1.router import (
     RSLUG_PREFIX,
     api,
 )
+from nucliadb.writer.api.v1.slug import ensure_slug_uniqueness, noop_context_manager
 from nucliadb.writer.back_pressure import maybe_back_pressure
-from nucliadb.writer.exceptions import IngestNotAvailable
 from nucliadb.writer.resource.audit import parse_audit
 from nucliadb.writer.resource.basic import (
-    parse_basic,
+    parse_basic_creation,
     parse_basic_modify,
-    set_processing_info,
     set_status,
     set_status_modify,
 )
 from nucliadb.writer.resource.field import extract_fields, parse_fields
 from nucliadb.writer.resource.origin import parse_extra, parse_origin
-from nucliadb.writer.resource.slug import resource_slug_exists
-from nucliadb.writer.resource.vectors import (
-    create_vectorset,
-    get_vectorsets,
-    parse_vectors,
-)
 from nucliadb.writer.utilities import get_processing
 from nucliadb_models.resource import NucliaDBRoles
 from nucliadb_models.writer import (
@@ -80,6 +63,8 @@ from nucliadb_models.writer import (
     ResourceUpdated,
     UpdateResourcePayload,
 )
+from nucliadb_protos.resources_pb2 import Metadata
+from nucliadb_protos.writer_pb2 import BrokerMessage, IndexResource
 from nucliadb_telemetry.errors import capture_exception
 from nucliadb_utils.authentication import requires
 from nucliadb_utils.exceptions import LimitsExceededError, SendToProcessError
@@ -87,7 +72,6 @@ from nucliadb_utils.utilities import (
     get_ingest,
     get_partitioning,
     get_storage,
-    get_transaction_utility,
 )
 
 
@@ -107,11 +91,17 @@ async def create_resource(
     item: CreateResourcePayload,
     kbid: str,
     x_skip_store: bool = SKIP_STORE_DEFAULT,
-    x_synchronous: bool = SYNC_CALL,
+    x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
+    kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+    if item.hidden and not (kb_config and kb_config.hidden_resources_enabled):
+        raise HTTPException(
+            status_code=422,
+            detail="Cannot hide a resource: the KB does not have hidden resources enabled",
+        )
+
     await maybe_back_pressure(request, kbid)
 
-    transaction = get_transaction_utility()
     partitioning = get_partitioning()
 
     # Create resource message
@@ -123,7 +113,7 @@ async def create_resource(
         uuid=uuid,
         kbid=kbid,
         partition=partition,
-        userid=request.headers.get("X-NUCLIADB-USER", ""),
+        userid=x_nucliadb_user,
         processing_options=item.processing_options,
     )
 
@@ -134,75 +124,52 @@ async def create_resource(
     toprocess.source = Source.HTTP
     toprocess.title = item.title
 
+    unique_slug_context_manager = noop_context_manager()
     if item.slug:
-        if await resource_slug_exists(kbid, item.slug):
-            raise HTTPException(
-                status_code=409, detail=f"Resource slug {item.slug} already exists"
-            )
+        unique_slug_context_manager = ensure_slug_uniqueness(kbid, item.slug)
         writer.slug = item.slug
         toprocess.slug = item.slug
 
-    parse_audit(writer.audit, request)
-    parse_basic(writer, item, toprocess)
+    async with unique_slug_context_manager:
+        parse_audit(writer.audit, request)
+        parse_basic_creation(writer, item, toprocess, kb_config)
+
+        if item.origin is not None:
+            parse_origin(writer.origin, item.origin)
+        if item.extra is not None:
+            parse_extra(writer.extra, item.extra)
+
+        await parse_fields(
+            writer=writer,
+            item=item,
+            toprocess=toprocess,
+            kbid=kbid,
+            uuid=uuid,
+            x_skip_store=x_skip_store,
+        )
 
-    if item.origin is not None:
-        parse_origin(writer.origin, item.origin)
-    if item.extra is not None:
-        parse_extra(writer.extra, item.extra)
+        set_status(writer.basic, item)
 
-    await parse_fields(
-        writer=writer,
-        item=item,
-        toprocess=toprocess,
-        kbid=kbid,
-        uuid=uuid,
-        x_skip_store=x_skip_store,
-    )
+        writer.source = BrokerMessage.MessageSource.WRITER
 
-    if item.uservectors:
-        vectorsets = await get_vectorsets(kbid)
-        if vectorsets and len(vectorsets.vectorsets):
-            parse_vectors(writer, item.uservectors, vectorsets)
+        if item.wait_for_commit:
+            t0 = time()
+        await transaction.commit(writer, partition, wait=item.wait_for_commit)
+
+        if item.wait_for_commit:
+            txn_time = time() - t0
         else:
-            for vector in item.uservectors:
-                if vector.vectors is not None:
-                    for vectorset, uservector in vector.vectors.items():
-                        if len(uservector) == 0:
-                            raise HTTPException(
-                                status_code=412,
-                                detail=str("Vectorset without vector not allowed"),
-                            )
-                        first_vector = list(uservector.values())[0]
-                        await create_vectorset(
-                            kbid, vectorset, len(first_vector.vector)
-                        )
-            vectorsets = await get_vectorsets(kbid)
-            if vectorsets is None or len(vectorsets.vectorsets) == 0:
-                raise HTTPException(
-                    status_code=412,
-                    detail=str("Vectorset was not able to be created"),
-                )
-            parse_vectors(writer, item.uservectors, vectorsets)
-
-    set_status(writer.basic, item)
-
-    seqid = await maybe_send_to_process(writer, toprocess, partition)
+            txn_time = None
 
-    writer.source = BrokerMessage.MessageSource.WRITER
-    if x_synchronous:
-        t0 = time()
-    await transaction.commit(writer, partition, wait=x_synchronous)
+        seqid = await maybe_send_to_process(toprocess, partition)
 
-    if x_synchronous:
-        return ResourceCreated(seqid=seqid, uuid=uuid, elapsed=time() - t0)
-    else:
-        return ResourceCreated(seqid=seqid, uuid=uuid)
+        return ResourceCreated(seqid=seqid, uuid=uuid, elapsed=txn_time)
 
 
 @api.patch(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}",
     status_code=200,
-    name="Modify Resource (by slug)",
+    summary="Modify Resource (by slug)",
     response_model=ResourceUpdated,
     tags=["Resources"],
 )
@@ -214,16 +181,15 @@ async def modify_resource_rslug_prefix(
     rslug: str,
     item: UpdateResourcePayload,
     x_skip_store: bool = SKIP_STORE_DEFAULT,
-    x_synchronous: bool = SYNC_CALL,
     x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
+    rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
     return await modify_resource_endpoint(
         request,
         item,
         kbid,
-        path_rslug=rslug,
+        rid,
         x_skip_store=x_skip_store,
-        x_synchronous=x_synchronous,
         x_nucliadb_user=x_nucliadb_user,
     )
 
@@ -231,7 +197,7 @@ async def modify_resource_rslug_prefix(
 @api.patch(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}",
     status_code=200,
-    name="Modify Resource (by id)",
+    summary="Modify Resource (by id)",
     response_model=ResourceUpdated,
     tags=["Resources"],
 )
@@ -243,16 +209,14 @@ async def modify_resource_rid_prefix(
     rid: str,
     item: UpdateResourcePayload,
     x_skip_store: bool = SKIP_STORE_DEFAULT,
-    x_synchronous: bool = SYNC_CALL,
     x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
     return await modify_resource_endpoint(
         request,
         item,
         kbid,
-        path_rid=rid,
+        rid,
         x_skip_store=x_skip_store,
-        x_synchronous=x_synchronous,
         x_nucliadb_user=x_nucliadb_user,
     )
 
@@ -261,15 +225,13 @@ async def modify_resource_endpoint(
     request: Request,
     item: UpdateResourcePayload,
     kbid: str,
+    rid: str,
     x_skip_store: bool,
-    x_synchronous: bool,
     x_nucliadb_user: str,
-    path_rid: Optional[str] = None,
-    path_rslug: Optional[str] = None,
 ):
-    resource_uuid = await get_rid_from_params_or_raise_error(kbid, path_rid, path_rslug)
+    await validate_rid_exists_or_raise_error(kbid, rid)
 
-    await maybe_back_pressure(request, kbid, resource_uuid=resource_uuid)
+    await maybe_back_pressure(request, kbid, resource_uuid=rid)
 
     if item.slug is None:
         return await modify_resource(
@@ -277,22 +239,18 @@ async def modify_resource_endpoint(
             item,
             kbid,
             x_skip_store=x_skip_store,
-            x_synchronous=x_synchronous,
             x_nucliadb_user=x_nucliadb_user,
-            rid=resource_uuid,
+            rid=rid,
         )
 
-    async with safe_update_resource_slug(
-        request, kbid, rid=resource_uuid, new_slug=item.slug
-    ):
+    async with safe_update_resource_slug(request, kbid, rid=rid, new_slug=item.slug):
         return await modify_resource(
             request,
             item,
             kbid,
             x_skip_store=x_skip_store,
-            x_synchronous=x_synchronous,
             x_nucliadb_user=x_nucliadb_user,
-            rid=resource_uuid,
+            rid=rid,
         )
 
 
@@ -301,12 +259,17 @@ async def modify_resource(
     item: UpdateResourcePayload,
     kbid: str,
     x_skip_store: bool,
-    x_synchronous: bool,
     x_nucliadb_user: str,
     *,
     rid: str,
 ):
-    transaction = get_transaction_utility()
+    kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+    if item.hidden and not (kb_config and kb_config.hidden_resources_enabled):
+        raise HTTPException(
+            status_code=422,
+            detail="Cannot hide a resource: the KB does not have hidden resources enabled",
+        )
+
     partitioning = get_partitioning()
 
     partition = partitioning.generate_partition(kbid, rid)
@@ -341,23 +304,16 @@ async def modify_resource(
         uuid=rid,
         x_skip_store=x_skip_store,
     )
-    if item.uservectors:
-        vectorsets = await get_vectorsets(kbid)
-        if vectorsets:
-            parse_vectors(writer, item.uservectors, vectorsets)
-        else:
-            raise HTTPException(status_code=412, detail=str("No vectorsets found"))
-
     set_status_modify(writer.basic, item)
 
     toprocess.title = writer.basic.title
-    seqid = await maybe_send_to_process(writer, toprocess, partition)
 
     writer.source = BrokerMessage.MessageSource.WRITER
 
     maybe_mark_reindex(writer, item)
 
-    await transaction.commit(writer, partition, wait=x_synchronous)
+    await transaction.commit(writer, partition)
+    seqid = await maybe_send_to_process(toprocess, partition)
 
     return ResourceUpdated(seqid=seqid)
 
@@ -397,9 +353,7 @@ async def update_resource_slug(
     new_slug: str,
 ):
     async with driver.transaction() as txn:
-        old_slug = await datamanagers.resources.modify_slug(
-            txn, kbid=kbid, rid=rid, new_slug=new_slug
-        )
+        old_slug = await datamanagers.resources.modify_slug(txn, kbid=kbid, rid=rid, new_slug=new_slug)
         await txn.commit()
         return old_slug
 
@@ -407,7 +361,7 @@ async def update_resource_slug(
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}/reprocess",
     status_code=202,
-    name="Reprocess resource (by slug)",
+    summary="Reprocess resource (by slug)",
     response_model=ResourceUpdated,
     tags=["Resources"],
 )
@@ -419,15 +373,14 @@ async def reprocess_resource_rslug_prefix(
     rslug: str,
     x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
-    return await _reprocess_resource(
-        request, kbid, rslug=rslug, x_nucliadb_user=x_nucliadb_user
-    )
+    rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
+    return await _reprocess_resource(request, kbid, rid, x_nucliadb_user=x_nucliadb_user)
 
 
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}/reprocess",
     status_code=202,
-    name="Reprocess resource (by id)",
+    summary="Reprocess resource (by id)",
     response_model=ResourceUpdated,
     tags=["Resources"],
 )
@@ -439,25 +392,20 @@ async def reprocess_resource_rid_prefix(
     rid: str,
     x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
-    return await _reprocess_resource(
-        request, kbid, rid=rid, x_nucliadb_user=x_nucliadb_user
-    )
+    return await _reprocess_resource(request, kbid, rid, x_nucliadb_user=x_nucliadb_user)
 
 
 async def _reprocess_resource(
     request: Request,
     kbid: str,
+    rid: str,
     x_nucliadb_user: str,
-    rid: Optional[str] = None,
-    rslug: Optional[str] = None,
 ):
-    transaction = get_transaction_utility()
-    partitioning = get_partitioning()
-
-    rid = await get_rid_from_params_or_raise_error(kbid, rid, rslug)
-
+    await validate_rid_exists_or_raise_error(kbid, rid)
     await maybe_back_pressure(request, kbid, resource_uuid=rid)
 
+    partitioning = get_partitioning()
+
     partition = partitioning.generate_partition(kbid, rid)
 
     toprocess = PushPayload(
@@ -483,16 +431,14 @@ async def _reprocess_resource(
 
         await extract_fields(resource=resource, toprocess=toprocess)
 
-    processing_info = await send_to_process(toprocess, partition)
-
     writer = BrokerMessage()
     writer.kbid = kbid
     writer.uuid = rid
     writer.source = BrokerMessage.MessageSource.WRITER
     writer.basic.metadata.useful = True
     writer.basic.metadata.status = Metadata.Status.PENDING
-    set_processing_info(writer, processing_info)
     await transaction.commit(writer, partition, wait=False)
+    processing_info = await send_to_process(toprocess, partition)
 
     return ResourceUpdated(seqid=processing_info.seqid)
 
@@ -500,7 +446,7 @@ async def _reprocess_resource(
 @api.delete(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}",
     status_code=204,
-    name="Delete Resource (by slug)",
+    summary="Delete Resource (by slug)",
     tags=["Resources"],
 )
 @requires(NucliaDBRoles.WRITER)
@@ -509,17 +455,15 @@ async def delete_resource_rslug_prefix(
     request: Request,
     kbid: str,
     rslug: str,
-    x_synchronous: bool = SYNC_CALL,
 ):
-    return await _delete_resource(
-        request, kbid, rslug=rslug, x_synchronous=x_synchronous
-    )
+    rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
+    return await _delete_resource(request, kbid, rid)
 
 
 @api.delete(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}",
     status_code=204,
-    name="Delete Resource (by id)",
+    summary="Delete Resource (by id)",
     tags=["Resources"],
 )
 @requires(NucliaDBRoles.WRITER)
@@ -528,22 +472,18 @@ async def delete_resource_rid_prefix(
     request: Request,
     kbid: str,
     rid: str,
-    x_synchronous: bool = SYNC_CALL,
 ):
-    return await _delete_resource(request, kbid, rid=rid, x_synchronous=x_synchronous)
+    return await _delete_resource(request, kbid, rid)
 
 
 async def _delete_resource(
     request: Request,
     kbid: str,
-    x_synchronous: bool,
-    rid: Optional[str] = None,
-    rslug: Optional[str] = None,
+    rid: str,
 ):
-    transaction = get_transaction_utility()
-    partitioning = get_partitioning()
+    await validate_rid_exists_or_raise_error(kbid, rid)
 
-    rid = await get_rid_from_params_or_raise_error(kbid, rid, rslug)
+    partitioning = get_partitioning()
 
     partition = partitioning.generate_partition(kbid, rid)
     writer = BrokerMessage()
@@ -553,10 +493,7 @@ async def _delete_resource(
     writer.type = BrokerMessage.MessageType.DELETE
 
     parse_audit(writer.audit, request)
-
-    # Create processing message
-    await transaction.commit(writer, partition, wait=x_synchronous)
-
+    await transaction.commit(writer, partition)
     processing = get_processing()
     asyncio.create_task(processing.delete_from_processing(kbid=kbid, resource_id=rid))
 
@@ -566,7 +503,7 @@ async def _delete_resource(
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}/reindex",
     status_code=204,
-    name="Reindex Resource (by slug)",
+    summary="Reindex Resource (by slug)",
     tags=["Resources"],
 )
 @requires(NucliaDBRoles.WRITER)
@@ -577,15 +514,14 @@ async def reindex_resource_rslug_prefix(
     rslug: str,
     reindex_vectors: bool = Query(False),
 ):
-    return await _reindex_resource(
-        request, kbid, rslug=rslug, reindex_vectors=reindex_vectors
-    )
+    rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
+    return await _reindex_resource(request, kbid, rid, reindex_vectors=reindex_vectors)
 
 
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}/reindex",
     status_code=204,
-    name="Reindex Resource (by id)",
+    summary="Reindex Resource (by id)",
     tags=["Resources"],
 )
 @requires(NucliaDBRoles.WRITER)
@@ -596,20 +532,16 @@ async def reindex_resource_rid_prefix(
     rid: str,
     reindex_vectors: bool = Query(False),
 ):
-    return await _reindex_resource(
-        request, kbid, rid=rid, reindex_vectors=reindex_vectors
-    )
+    return await _reindex_resource(request, kbid, rid, reindex_vectors=reindex_vectors)
 
 
 async def _reindex_resource(
     request: Request,
     kbid: str,
+    rid: str,
     reindex_vectors: bool,
-    rid: Optional[str] = None,
-    rslug: Optional[str] = None,
 ):
-    rid = await get_rid_from_params_or_raise_error(kbid, rid, rslug)
-
+    await validate_rid_exists_or_raise_error(kbid, rid)
     await maybe_back_pressure(request, kbid, resource_uuid=rid)
 
     ingest = get_ingest()
@@ -622,55 +554,18 @@ async def _reindex_resource(
     return Response(status_code=200)
 
 
-async def get_resource_uuid_from_slug(kbid: str, slug: str) -> str:
-    ingest = get_ingest()
-    pbrequest = ResourceIdRequest()
-    pbrequest.kbid = kbid
-    pbrequest.slug = slug
-    try:
-        response: ResourceIdResponse = await ingest.GetResourceId(pbrequest)  # type: ignore
-    except AioRpcError as exc:
-        if exc.code() is GrpcStatusCode.UNAVAILABLE:
-            raise IngestNotAvailable()
-        else:
-            raise exc
-    return response.uuid
-
-
-async def get_rid_from_params_or_raise_error(
-    kbid: str,
-    rid: Optional[str] = None,
-    slug: Optional[str] = None,
-) -> str:
-    if rid is not None:
-        ingest = get_ingest()
-        pbrequest = ResourceFieldId()
-        pbrequest.kbid = kbid
-        pbrequest.rid = rid
-
-        try:
-            response: ResourceFieldExistsResponse = await ingest.ResourceFieldExists(pbrequest)  # type: ignore
-        except AioRpcError as exc:
-            if exc.code() is GrpcStatusCode.UNAVAILABLE:
-                raise IngestNotAvailable()
-            else:
-                raise exc
-
-        if response.found:
-            return rid
-        else:
-            raise HTTPException(status_code=404, detail="Resource does not exist")
-
-    if slug is None:
-        raise ValueError("Either rid or slug must be set")
-
-    rid = await get_resource_uuid_from_slug(kbid, slug)
+async def get_rid_from_slug_or_raise_error(kbid: str, rslug: str) -> str:
+    rid = await datamanagers.atomic.resources.get_resource_uuid_from_slug(kbid=kbid, slug=rslug)
     if not rid:
         raise HTTPException(status_code=404, detail="Resource does not exist")
-
     return rid
 
 
+async def validate_rid_exists_or_raise_error(kbid: str, rid: str):
+    if not (await datamanagers.atomic.resources.resource_exists(kbid=kbid, rid=rid)):
+        raise HTTPException(status_code=404, detail="Resource does not exist")
+
+
 def maybe_mark_reindex(message: BrokerMessage, item: UpdateResourcePayload):
     if needs_resource_reindex(item):
         message.reindex = True
@@ -681,24 +576,25 @@ def needs_resource_reindex(item: UpdateResourcePayload) -> bool:
     # a resource and that means this message should force reindexing everything.
     # XXX This is not ideal. Long term, we should handle it differently
     # so this is not required
-    return item.usermetadata is not None or (
-        item.origin is not None
-        and (
-            item.origin.created is not None
-            or item.origin.modified is not None
-            or item.origin.metadata is not None
+    return (
+        item.usermetadata is not None
+        or item.hidden is not None
+        or (
+            item.origin is not None
+            and (
+                item.origin.created is not None
+                or item.origin.modified is not None
+                or item.origin.metadata is not None
+            )
         )
     )
 
 
-async def maybe_send_to_process(
-    writer: BrokerMessage, toprocess: PushPayload, partition
-) -> Optional[int]:
+async def maybe_send_to_process(toprocess: PushPayload, partition) -> Optional[int]:
     if not needs_reprocess(toprocess):
         return None
 
     processing_info = await send_to_process(toprocess, partition)
-    set_processing_info(writer, processing_info)
     return processing_info.seqid
 
 
@@ -710,7 +606,10 @@ async def send_to_process(toprocess: PushPayload, partition) -> ProcessingInfo:
     except LimitsExceededError as exc:
         raise HTTPException(status_code=exc.status_code, detail=exc.detail)
     except SendToProcessError:
-        raise HTTPException(status_code=500, detail="Error while sending to process")
+        raise HTTPException(
+            status_code=500,
+            detail="Error while sending to process. Try calling /reprocess",
+        )
 
 
 def needs_reprocess(processing_payload: PushPayload) -> bool:
@@ -724,7 +623,6 @@ def needs_reprocess(processing_payload: PushPayload) -> bool:
         "filefield",
         "linkfield",
         "textfield",
-        "layoutfield",
         "conversationfield",
     ):
         if len(getattr(processing_payload, field)) > 0: