nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/writer/api/v1/knowledgebox.py

@@ -18,75 +18,142 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import asyncio
-import json
+from functools import partial
 
 from fastapi import HTTPException, Response
 from fastapi_versioning import version
-from nucliadb_protos.knowledgebox_pb2 import (
-    DeleteKnowledgeBoxResponse,
-    KnowledgeBoxID,
-    KnowledgeBoxNew,
-    KnowledgeBoxResponseStatus,
-    KnowledgeBoxUpdate,
-    NewKnowledgeBoxResponse,
-    UpdateKnowledgeBoxResponse,
-)
 from starlette.requests import Request
 
+from nucliadb import learning_proxy
+from nucliadb.common import datamanagers
+from nucliadb.common.external_index_providers.exceptions import (
+    ExternalIndexCreationError,
+)
+from nucliadb.common.maindb.utils import get_driver
+from nucliadb.ingest.orm.exceptions import KnowledgeBoxConflict
+from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
+from nucliadb.writer import logger, vectorsets
+from nucliadb.writer.api.utils import only_for_onprem
 from nucliadb.writer.api.v1.router import KB_PREFIX, KBS_PREFIX, api
 from nucliadb.writer.utilities import get_processing
+from nucliadb_models.external_index_providers import (
+    ExternalIndexProviderType,
+    PineconeServerlessCloud,
+)
 from nucliadb_models.resource import (
     KnowledgeBoxConfig,
     KnowledgeBoxObj,
     KnowledgeBoxObjID,
     NucliaDBRoles,
 )
+from nucliadb_protos import knowledgebox_pb2
 from nucliadb_utils.authentication import requires
-from nucliadb_utils.utilities import get_ingest
 
 
+@only_for_onprem
 @api.post(
     f"/{KBS_PREFIX}",
     status_code=201,
     summary="Create Knowledge Box",
-    response_model=KnowledgeBoxObj,
     tags=["Knowledge Boxes"],
     openapi_extra={"x-hidden-operation": True},
 )
 @requires(NucliaDBRoles.MANAGER)
 @version(1)
-async def create_kb(request: Request, item: KnowledgeBoxConfig):
-    ingest = get_ingest()
-    requestpb = KnowledgeBoxNew()
-    requestpb = parse_create_kb_request(item)
-    kbobj: NewKnowledgeBoxResponse = await ingest.NewKnowledgeBox(requestpb)  # type: ignore
-    if item.slug != "":
-        slug = item.slug
-    else:
-        slug = kbobj.uuid  # type: ignore
-    if kbobj.status == KnowledgeBoxResponseStatus.OK:
-        return KnowledgeBoxObj(uuid=kbobj.uuid, slug=slug)
-    elif kbobj.status == KnowledgeBoxResponseStatus.CONFLICT:
+async def create_kb_endpoint(request: Request, item: KnowledgeBoxConfig) -> KnowledgeBoxObj:
+    try:
+        kbid, slug = await create_kb(item)
+    except KnowledgeBoxConflict:
         raise HTTPException(status_code=419, detail="Knowledge box already exists")
-    elif kbobj.status == KnowledgeBoxResponseStatus.ERROR:
-        raise HTTPException(status_code=500, detail="Error on creating knowledge box")
-
-
-def parse_create_kb_request(item: KnowledgeBoxConfig) -> KnowledgeBoxNew:
-    requestpb = KnowledgeBoxNew()
-    if item.slug:
-        requestpb.slug = item.slug
-    if item.title:
-        requestpb.config.title = item.title
-    if item.description:
-        requestpb.config.description = item.description
-    if item.release_channel:
-        requestpb.release_channel = item.release_channel.to_pb()
+    except ExternalIndexCreationError as exc:
+        raise HTTPException(status_code=502, detail=str(exc))
+    except Exception:
+        logger.exception("Could not create KB")
+        raise HTTPException(status_code=500, detail="Error creating knowledge box")
+    else:
+        return KnowledgeBoxObj(uuid=kbid, slug=slug)
+
+
+async def create_kb(item: KnowledgeBoxConfig) -> tuple[str, str]:
+    driver = get_driver()
+    rollback_learning_config = None
+
+    kbid = KnowledgeBox.new_unique_kbid()
+
+    # Onprem KB creation doesn't have an existing learning configuration yet, so
+    # we need to call learning proxy to create it
     if item.learning_configuration:
-        requestpb.learning_config = json.dumps(item.learning_configuration)
-    return requestpb
+        user_learning_config = item.learning_configuration
+    else:
+        logger.warning(
+            "No learning configuration provided. Default will be used.",
+            extra={"kbid": kbid},
+        )
+        # learning will choose the default values
+        user_learning_config = {}
+
+    # We need to be backward compatible with the old "semantic_model" field where
+    # only one semantic model was allowed.
+    if "semantic_model" in user_learning_config:
+        user_learning_config["semantic_models"] = [user_learning_config.pop("semantic_model")]
 
+    # we rely on learning to return the updated configuration with defaults and
+    # any other needed values (e.g. matryoshka settings if available)
+    learning_config = await learning_proxy.set_configuration(kbid, config=user_learning_config)
 
+    # if KB creation fails, we'll have to delete its learning config
+    async def _rollback_learning_config(kbid: str):
+        try:
+            await learning_proxy.delete_configuration(kbid)
+        except Exception:
+            logger.warning(
+                "Could not rollback learning configuration",
+                exc_info=True,
+                extra={"kbid": kbid},
+            )
+
+    rollback_learning_config = partial(_rollback_learning_config, kbid)
+    semantic_models = learning_config.into_semantic_models_metadata()
+
+    external_index_provider = knowledgebox_pb2.CreateExternalIndexProviderMetadata(
+        type=knowledgebox_pb2.ExternalIndexProviderType.UNSET,
+    )
+    if (
+        item.external_index_provider
+        and item.external_index_provider.type == ExternalIndexProviderType.PINECONE
+    ):
+        pinecone_api_key = item.external_index_provider.api_key
+        serverless_pb = to_pinecone_serverless_cloud_pb(item.external_index_provider.serverless_cloud)
+        external_index_provider = knowledgebox_pb2.CreateExternalIndexProviderMetadata(
+            type=knowledgebox_pb2.ExternalIndexProviderType.PINECONE,
+            pinecone_config=knowledgebox_pb2.CreatePineconeConfig(
+                api_key=pinecone_api_key,
+                serverless_cloud=serverless_pb,
+            ),
+        )
+
+    try:
+        (kbid, slug) = await KnowledgeBox.create(
+            driver,
+            kbid=kbid,
+            slug=item.slug or kbid,
+            title=item.title or "",
+            description=item.description or "",
+            semantic_models=semantic_models,
+            external_index_provider=external_index_provider,
+            hidden_resources_enabled=item.hidden_resources_enabled,
+            hidden_resources_hide_on_creation=item.hidden_resources_hide_on_creation,
+        )
+
+    except Exception as exc:
+        logger.error("Unexpected error creating KB", exc_info=exc, extra={"slug": item.slug})
+        await rollback_learning_config()
+        raise
+
+    return (kbid, slug)
+
+
+@only_for_onprem
 @api.patch(
     f"/{KB_PREFIX}/{{kbid}}",
     status_code=200,
@@ -97,48 +164,131 @@ def parse_create_kb_request(item: KnowledgeBoxConfig) -> KnowledgeBoxNew:
 )
 @requires(NucliaDBRoles.MANAGER)
 @version(1)
-async def update_kb(request: Request, kbid: str, item: KnowledgeBoxConfig):
-    ingest = get_ingest()
-    pbrequest = KnowledgeBoxUpdate(uuid=kbid)
-    if item.slug is not None:
-        pbrequest.slug = item.slug
-    if item.title:
-        pbrequest.config.title = item.title
-    if item.description:
-        pbrequest.config.description = item.description
-    kbobj: UpdateKnowledgeBoxResponse = await ingest.UpdateKnowledgeBox(pbrequest)  # type: ignore
-    if kbobj.status == KnowledgeBoxResponseStatus.OK:
-        return KnowledgeBoxObjID(uuid=kbobj.uuid)
-    elif kbobj.status == KnowledgeBoxResponseStatus.NOTFOUND:
+async def update_kb(request: Request, kbid: str, item: KnowledgeBoxConfig) -> KnowledgeBoxObjID:
+    driver = get_driver()
+    config = None
+    if (
+        item.slug
+        or item.title
+        or item.description
+        or item.hidden_resources_enabled
+        or item.hidden_resources_hide_on_creation
+    ):
+        config = knowledgebox_pb2.KnowledgeBoxConfig(
+            slug=item.slug or "",
+            title=item.title or "",
+            description=item.description or "",
+            hidden_resources_enabled=item.hidden_resources_enabled,
+            hidden_resources_hide_on_creation=item.hidden_resources_hide_on_creation,
+        )
+    try:
+        async with driver.transaction() as txn:
+            await KnowledgeBox.update(
+                txn,
+                uuid=kbid,
+                slug=item.slug,
+                config=config,
+            )
+            await txn.commit()
+    except datamanagers.exceptions.KnowledgeBoxNotFound:
         raise HTTPException(status_code=404, detail="Knowledge box does not exist")
-    elif kbobj.status == KnowledgeBoxResponseStatus.ERROR:
-        raise HTTPException(status_code=500, detail="Error on creating knowledge box")
+    except Exception as exc:
+        logger.exception("Could not update KB", exc_info=exc, extra={"kbid": kbid})
+        raise HTTPException(status_code=500, detail="Error updating knowledge box")
+    else:
+        return KnowledgeBoxObjID(uuid=kbid)
 
 
+@only_for_onprem
 @api.delete(
     f"/{KB_PREFIX}/{{kbid}}",
     status_code=200,
     summary="Delete Knowledge Box",
-    response_model=KnowledgeBoxObj,
     tags=["Knowledge Boxes"],
     openapi_extra={"x-hidden-operation": True},
 )
 @requires(NucliaDBRoles.MANAGER)
 @version(1)
-async def delete_kb(request: Request, kbid: str):
-    ingest = get_ingest()
-
-    kbobj: DeleteKnowledgeBoxResponse = await ingest.DeleteKnowledgeBox(  # type: ignore
-        KnowledgeBoxID(uuid=kbid)
-    )
-    if kbobj.status == KnowledgeBoxResponseStatus.OK:
-        return KnowledgeBoxObj(uuid=kbid)
-    elif kbobj.status == KnowledgeBoxResponseStatus.NOTFOUND:
+async def delete_kb(request: Request, kbid: str) -> KnowledgeBoxObj:
+    driver = get_driver()
+    try:
+        await KnowledgeBox.delete(driver, kbid=kbid)
+    except datamanagers.exceptions.KnowledgeBoxNotFound:
         raise HTTPException(status_code=404, detail="Knowledge Box does not exists")
-    elif kbobj.status == KnowledgeBoxResponseStatus.ERROR:
-        raise HTTPException(status_code=500, detail="Error on deleting knowledge box")
+    except Exception as exc:
+        logger.exception("Could not delete KB", exc_info=exc, extra={"kbid": kbid})
+        raise HTTPException(status_code=500, detail="Error deleting knowledge box")
+
+    # onprem nucliadb must delete its learning configuration
+    try:
+        await learning_proxy.delete_configuration(kbid)
+        logger.info("Learning configuration deleted", extra={"kbid": kbid})
+    except Exception as exc:
+        logger.exception(
+            "Unexpected error deleting learning configuration",
+            exc_info=exc,
+            extra={"kbid": kbid},
+        )
 
+    # be nice and notify processing this KB is being deleted so we waste
+    # resources
     processing = get_processing()
     asyncio.create_task(processing.delete_from_processing(kbid=kbid))
 
-    return Response(status_code=204)
+    return KnowledgeBoxObj(uuid=kbid)
+
+
+def to_pinecone_serverless_cloud_pb(
+    serverless: PineconeServerlessCloud,
+) -> knowledgebox_pb2.PineconeServerlessCloud.ValueType:
+    return {
+        PineconeServerlessCloud.AWS_EU_WEST_1: knowledgebox_pb2.PineconeServerlessCloud.AWS_EU_WEST_1,
+        PineconeServerlessCloud.AWS_US_EAST_1: knowledgebox_pb2.PineconeServerlessCloud.AWS_US_EAST_1,
+        PineconeServerlessCloud.AWS_US_WEST_2: knowledgebox_pb2.PineconeServerlessCloud.AWS_US_WEST_2,
+        PineconeServerlessCloud.AZURE_EASTUS2: knowledgebox_pb2.PineconeServerlessCloud.AZURE_EASTUS2,
+        PineconeServerlessCloud.GCP_US_CENTRAL1: knowledgebox_pb2.PineconeServerlessCloud.GCP_US_CENTRAL1,
+    }[serverless]
+
+
+@api.post(
+    f"/{KB_PREFIX}/{{kbid}}/vectorsets/{{vectorset_id}}",
+    status_code=200,
+    summary="Add a vectorset to Knowledge Box",
+    tags=["Knowledge Boxes"],
+    # TODO: remove when the feature is mature
+    include_in_schema=False,
+)
+@requires(NucliaDBRoles.MANAGER)
+@version(1)
+async def add_vectorset(request: Request, kbid: str, vectorset_id: str) -> Response:
+    try:
+        await vectorsets.add(kbid, vectorset_id)
+    except learning_proxy.ProxiedLearningConfigError as err:
+        return Response(
+            status_code=err.status_code,
+            content=err.content,
+            media_type=err.content_type,
+        )
+    return Response(status_code=200)
+
+
+@api.delete(
+    f"/{KB_PREFIX}/{{kbid}}/vectorsets/{{vectorset_id}}",
+    status_code=200,
+    summary="Delete vectorset from Knowledge Box",
+    tags=["Knowledge Boxes"],
+    # TODO: remove when the feature is mature
+    include_in_schema=False,
+)
+@requires(NucliaDBRoles.MANAGER)
+@version(1)
+async def delete_vectorset(request: Request, kbid: str, vectorset_id: str) -> Response:
+    try:
+        await vectorsets.delete(kbid, vectorset_id)
+    except learning_proxy.ProxiedLearningConfigError as err:
+        return Response(
+            status_code=err.status_code,
+            content=err.content,
+            media_type=err.content_type,
+        )
+    return Response(status_code=200)

nucliadb/writer/api/v1/resource.py

@@ -25,8 +25,6 @@ from uuid import uuid4
 
 from fastapi import HTTPException, Query, Response
 from fastapi_versioning import version
-from nucliadb_protos.resources_pb2 import Metadata
-from nucliadb_protos.writer_pb2 import BrokerMessage, IndexResource
 from starlette.requests import Request
 
 from nucliadb.common import datamanagers
@@ -38,6 +36,7 @@ from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
 from nucliadb.ingest.processing import ProcessingInfo, PushPayload, Source
 from nucliadb.writer import SERVICE_NAME, logger
 from nucliadb.writer.api.constants import SKIP_STORE_DEFAULT, X_NUCLIADB_USER
+from nucliadb.writer.api.v1 import transaction
 from nucliadb.writer.api.v1.router import (
     KB_PREFIX,
     RESOURCE_PREFIX,
@@ -45,10 +44,11 @@ from nucliadb.writer.api.v1.router import (
     RSLUG_PREFIX,
     api,
 )
+from nucliadb.writer.api.v1.slug import ensure_slug_uniqueness, noop_context_manager
 from nucliadb.writer.back_pressure import maybe_back_pressure
 from nucliadb.writer.resource.audit import parse_audit
 from nucliadb.writer.resource.basic import (
-    parse_basic,
+    parse_basic_creation,
     parse_basic_modify,
     set_status,
     set_status_modify,
@@ -63,15 +63,15 @@ from nucliadb_models.writer import (
     ResourceUpdated,
     UpdateResourcePayload,
 )
+from nucliadb_protos.resources_pb2 import Metadata
+from nucliadb_protos.writer_pb2 import BrokerMessage, IndexResource
 from nucliadb_telemetry.errors import capture_exception
 from nucliadb_utils.authentication import requires
 from nucliadb_utils.exceptions import LimitsExceededError, SendToProcessError
-from nucliadb_utils.transaction import TransactionCommitTimeoutError
 from nucliadb_utils.utilities import (
     get_ingest,
     get_partitioning,
     get_storage,
-    get_transaction_utility,
 )
 
 
@@ -91,10 +91,17 @@ async def create_resource(
     item: CreateResourcePayload,
     kbid: str,
     x_skip_store: bool = SKIP_STORE_DEFAULT,
+    x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
+    kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+    if item.hidden and not (kb_config and kb_config.hidden_resources_enabled):
+        raise HTTPException(
+            status_code=422,
+            detail="Cannot hide a resource: the KB does not have hidden resources enabled",
+        )
+
     await maybe_back_pressure(request, kbid)
 
-    transaction = get_transaction_utility()
     partitioning = get_partitioning()
 
     # Create resource message
@@ -106,7 +113,7 @@ async def create_resource(
         uuid=uuid,
         kbid=kbid,
         partition=partition,
-        userid=request.headers.get("X-NUCLIADB-USER", ""),
+        userid=x_nucliadb_user,
         processing_options=item.processing_options,
     )
 
@@ -117,49 +124,46 @@ async def create_resource(
     toprocess.source = Source.HTTP
     toprocess.title = item.title
 
+    unique_slug_context_manager = noop_context_manager()
     if item.slug:
-        from nucliadb.common import datamanagers
-
-        if await datamanagers.atomic.resources.slug_exists(kbid=kbid, slug=item.slug):
-            raise HTTPException(
-                status_code=409, detail=f"Resource slug {item.slug} already exists"
-            )
+        unique_slug_context_manager = ensure_slug_uniqueness(kbid, item.slug)
         writer.slug = item.slug
         toprocess.slug = item.slug
 
-    parse_audit(writer.audit, request)
-    parse_basic(writer, item, toprocess)
+    async with unique_slug_context_manager:
+        parse_audit(writer.audit, request)
+        parse_basic_creation(writer, item, toprocess, kb_config)
+
+        if item.origin is not None:
+            parse_origin(writer.origin, item.origin)
+        if item.extra is not None:
+            parse_extra(writer.extra, item.extra)
+
+        await parse_fields(
+            writer=writer,
+            item=item,
+            toprocess=toprocess,
+            kbid=kbid,
+            uuid=uuid,
+            x_skip_store=x_skip_store,
+        )
 
-    if item.origin is not None:
-        parse_origin(writer.origin, item.origin)
-    if item.extra is not None:
-        parse_extra(writer.extra, item.extra)
+        set_status(writer.basic, item)
 
-    await parse_fields(
-        writer=writer,
-        item=item,
-        toprocess=toprocess,
-        kbid=kbid,
-        uuid=uuid,
-        x_skip_store=x_skip_store,
-    )
+        writer.source = BrokerMessage.MessageSource.WRITER
 
-    set_status(writer.basic, item)
+        if item.wait_for_commit:
+            t0 = time()
+        await transaction.commit(writer, partition, wait=item.wait_for_commit)
 
-    writer.source = BrokerMessage.MessageSource.WRITER
-    try:
-        t0 = time()
-        await transaction.commit(writer, partition, wait=True)
-        txn_time = time() - t0
-    except TransactionCommitTimeoutError:
-        raise HTTPException(
-            status_code=501,
-            detail="Inconsistent write. This resource will not be processed and may not be stored.",
-        )
+        if item.wait_for_commit:
+            txn_time = time() - t0
+        else:
+            txn_time = None
 
-    seqid = await maybe_send_to_process(toprocess, partition)
+        seqid = await maybe_send_to_process(toprocess, partition)
 
-    return ResourceCreated(seqid=seqid, uuid=uuid, elapsed=txn_time)
+        return ResourceCreated(seqid=seqid, uuid=uuid, elapsed=txn_time)
 
 
 @api.patch(
@@ -259,7 +263,13 @@ async def modify_resource(
     *,
     rid: str,
 ):
-    transaction = get_transaction_utility()
+    kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+    if item.hidden and not (kb_config and kb_config.hidden_resources_enabled):
+        raise HTTPException(
+            status_code=422,
+            detail="Cannot hide a resource: the KB does not have hidden resources enabled",
+        )
+
     partitioning = get_partitioning()
 
     partition = partitioning.generate_partition(kbid, rid)
@@ -302,14 +312,7 @@ async def modify_resource(
 
     maybe_mark_reindex(writer, item)
 
-    try:
-        await transaction.commit(writer, partition, wait=True)
-    except TransactionCommitTimeoutError:
-        raise HTTPException(
-            status_code=501,
-            detail="Inconsistent write. This resource will not be processed and may not be stored.",
-        )
-
+    await transaction.commit(writer, partition)
     seqid = await maybe_send_to_process(toprocess, partition)
 
     return ResourceUpdated(seqid=seqid)
@@ -350,9 +353,7 @@ async def update_resource_slug(
     new_slug: str,
 ):
     async with driver.transaction() as txn:
-        old_slug = await datamanagers.resources.modify_slug(
-            txn, kbid=kbid, rid=rid, new_slug=new_slug
-        )
+        old_slug = await datamanagers.resources.modify_slug(txn, kbid=kbid, rid=rid, new_slug=new_slug)
         await txn.commit()
         return old_slug
 
@@ -373,9 +374,7 @@ async def reprocess_resource_rslug_prefix(
     x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
     rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
-    return await _reprocess_resource(
-        request, kbid, rid, x_nucliadb_user=x_nucliadb_user
-    )
+    return await _reprocess_resource(request, kbid, rid, x_nucliadb_user=x_nucliadb_user)
 
 
 @api.post(
@@ -393,9 +392,7 @@ async def reprocess_resource_rid_prefix(
     rid: str,
     x_nucliadb_user: str = X_NUCLIADB_USER,
 ):
-    return await _reprocess_resource(
-        request, kbid, rid, x_nucliadb_user=x_nucliadb_user
-    )
+    return await _reprocess_resource(request, kbid, rid, x_nucliadb_user=x_nucliadb_user)
 
 
 async def _reprocess_resource(
@@ -407,7 +404,6 @@ async def _reprocess_resource(
     await validate_rid_exists_or_raise_error(kbid, rid)
     await maybe_back_pressure(request, kbid, resource_uuid=rid)
 
-    transaction = get_transaction_utility()
     partitioning = get_partitioning()
 
     partition = partitioning.generate_partition(kbid, rid)
@@ -441,14 +437,7 @@ async def _reprocess_resource(
     writer.source = BrokerMessage.MessageSource.WRITER
     writer.basic.metadata.useful = True
     writer.basic.metadata.status = Metadata.Status.PENDING
-    try:
-        await transaction.commit(writer, partition, wait=False)
-    except TransactionCommitTimeoutError:
-        raise HTTPException(
-            status_code=501,
-            detail="Inconsistent write. This resource will not be processed and may not be stored.",
-        )
-
+    await transaction.commit(writer, partition, wait=False)
     processing_info = await send_to_process(toprocess, partition)
 
     return ResourceUpdated(seqid=processing_info.seqid)
@@ -494,7 +483,6 @@ async def _delete_resource(
 ):
     await validate_rid_exists_or_raise_error(kbid, rid)
 
-    transaction = get_transaction_utility()
     partitioning = get_partitioning()
 
     partition = partitioning.generate_partition(kbid, rid)
@@ -505,16 +493,7 @@ async def _delete_resource(
     writer.type = BrokerMessage.MessageType.DELETE
 
     parse_audit(writer.audit, request)
-
-    # Create processing message
-    try:
-        await transaction.commit(writer, partition, wait=True)
-    except TransactionCommitTimeoutError:
-        raise HTTPException(
-            status_code=501,
-            detail="Inconsistent write. This resource will not be processed and may not be stored.",
-        )
-
+    await transaction.commit(writer, partition)
     processing = get_processing()
     asyncio.create_task(processing.delete_from_processing(kbid=kbid, resource_id=rid))
 
@@ -576,9 +555,7 @@ async def _reindex_resource(
 
 
 async def get_rid_from_slug_or_raise_error(kbid: str, rslug: str) -> str:
-    rid = await datamanagers.atomic.resources.get_resource_uuid_from_slug(
-        kbid=kbid, slug=rslug
-    )
+    rid = await datamanagers.atomic.resources.get_resource_uuid_from_slug(kbid=kbid, slug=rslug)
     if not rid:
         raise HTTPException(status_code=404, detail="Resource does not exist")
     return rid
@@ -599,12 +576,16 @@ def needs_resource_reindex(item: UpdateResourcePayload) -> bool:
     # a resource and that means this message should force reindexing everything.
     # XXX This is not ideal. Long term, we should handle it differently
     # so this is not required
-    return item.usermetadata is not None or (
-        item.origin is not None
-        and (
-            item.origin.created is not None
-            or item.origin.modified is not None
-            or item.origin.metadata is not None
+    return (
+        item.usermetadata is not None
+        or item.hidden is not None
+        or (
+            item.origin is not None
+            and (
+                item.origin.created is not None
+                or item.origin.modified is not None
+                or item.origin.metadata is not None
+            )
         )
     )
 
@@ -642,7 +623,6 @@ def needs_reprocess(processing_payload: PushPayload) -> bool:
         "filefield",
         "linkfield",
         "textfield",
-        "layoutfield",
         "conversationfield",
     ):
         if len(getattr(processing_payload, field)) > 0: