nucliadb 2.46.1.post382__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/writer/api/v1/upload.py

@@ -18,7 +18,6 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import base64
-import mimetypes
 import pickle
 import uuid
 from datetime import datetime
@@ -30,30 +29,23 @@ from fastapi import HTTPException
 from fastapi.params import Header
 from fastapi.requests import Request
 from fastapi.responses import Response
-from fastapi_versioning import version  # type: ignore
-from grpc import StatusCode as GrpcStatusCode
-from grpc.aio import AioRpcError
-from nucliadb_protos.resources_pb2 import FieldFile
-from nucliadb_protos.writer_pb2 import (
-    BrokerMessage,
-    ResourceFieldExistsResponse,
-    ResourceFieldId,
-)
+from fastapi_versioning import version
 from starlette.requests import Request as StarletteRequest
 
+from nucliadb.common import datamanagers
 from nucliadb.ingest.orm.utils import set_title
 from nucliadb.ingest.processing import PushPayload, Source
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.writer import SERVICE_NAME
-from nucliadb.writer.api.v1.resource import get_rid_from_params_or_raise_error
-from nucliadb.writer.back_pressure import maybe_back_pressure
-from nucliadb.writer.exceptions import (
-    ConflictError,
-    IngestNotAvailable,
-    ResourceNotFound,
+from nucliadb.writer.api.v1 import transaction
+from nucliadb.writer.api.v1.resource import (
+    get_rid_from_slug_or_raise_error,
+    validate_rid_exists_or_raise_error,
 )
+from nucliadb.writer.api.v1.slug import ensure_slug_uniqueness, noop_context_manager
+from nucliadb.writer.back_pressure import maybe_back_pressure
 from nucliadb.writer.resource.audit import parse_audit
-from nucliadb.writer.resource.basic import parse_basic, set_processing_info
+from nucliadb.writer.resource.basic import parse_basic_creation
 from nucliadb.writer.resource.field import parse_fields
 from nucliadb.writer.resource.origin import parse_extra, parse_origin
 from nucliadb.writer.tus import TUSUPLOAD, UPLOAD, get_dm, get_storage_manager
@@ -62,23 +54,24 @@ from nucliadb.writer.tus.exceptions import (
     HTTPConflict,
     HTTPNotFound,
     HTTPPreconditionFailed,
-    HTTPServiceUnavailable,
     InvalidTUSMetadata,
     ResumableURINotAvailable,
 )
-from nucliadb.writer.tus.storage import FileStorageManager  # type: ignore
+from nucliadb.writer.tus.storage import FileStorageManager
 from nucliadb.writer.tus.utils import parse_tus_metadata
 from nucliadb.writer.utilities import get_processing
+from nucliadb_models import content_types
 from nucliadb_models.resource import NucliaDBRoles
+from nucliadb_models.utils import FieldIdString
 from nucliadb_models.writer import CreateResourcePayload, ResourceFileUploaded
+from nucliadb_protos.resources_pb2 import CloudFile, FieldFile, Metadata
+from nucliadb_protos.writer_pb2 import BrokerMessage
 from nucliadb_utils.authentication import requires_one
 from nucliadb_utils.exceptions import LimitsExceededError, SendToProcessError
 from nucliadb_utils.storages.storage import KB_RESOURCE_FIELD
 from nucliadb_utils.utilities import (
-    get_ingest,
     get_partitioning,
     get_storage,
-    get_transaction_utility,
 )
 
 from .router import KB_PREFIX, RESOURCE_PREFIX, RSLUG_PREFIX, api
@@ -105,14 +98,14 @@ TUS_HEADERS = {
 @api.options(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}/file/{{field}}/{TUSUPLOAD}",
     tags=["Resource field TUS uploads"],
-    name="TUS Server information",
+    summary="TUS Server information",
     openapi_extra={"x-operation-order": 4},
     include_in_schema=False,
 )
 @api.options(
     f"/{KB_PREFIX}/{{kbid}}/{TUSUPLOAD}",
     tags=["Knowledge Box TUS uploads"],
-    name="TUS Server information",
+    summary="TUS Server information",
     openapi_extra={"x-operation-order": 4},
 )
 @version(1)
@@ -138,7 +131,7 @@ def _tus_options() -> Response:
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}/file/{{field}}/{TUSUPLOAD}",
     tags=["Resource field TUS uploads"],
-    name="Create new upload on a Resource (by slug)",
+    summary="Create new upload on a Resource (by slug)",
     openapi_extra={"x-operation-order": 1},
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -147,16 +140,17 @@ async def tus_post_rslug_prefix(
     request: Request,
     kbid: str,
     rslug: str,
-    field: str,
+    field: FieldIdString,
     item: Optional[CreateResourcePayload] = None,
 ) -> Response:
-    return await _tus_post(request, kbid, item=item, rslug=rslug, field=field)
+    rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
+    return await _tus_post(request, kbid, item, path_rid=rid, field_id=field)
 
 
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{path_rid}}/file/{{field}}/{TUSUPLOAD}",
     tags=["Resource field TUS uploads"],
-    name="Create new upload on a Resource (by id)",
+    summary="Create new upload on a Resource (by id)",
     openapi_extra={"x-operation-order": 1},
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -165,16 +159,16 @@ async def tus_post_rid_prefix(
     request: Request,
     kbid: str,
     path_rid: str,
-    field: str,
+    field: FieldIdString,
     item: Optional[CreateResourcePayload] = None,
 ) -> Response:
-    return await _tus_post(request, kbid, item=item, path_rid=path_rid, field=field)
+    return await _tus_post(request, kbid, item, path_rid=path_rid, field_id=field)
 
 
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{TUSUPLOAD}",
     tags=["Knowledge Box TUS uploads"],
-    name="Create new upload on a Knowledge Box",
+    summary="Create new upload on a Knowledge Box",
     openapi_extra={"x-operation-order": 1},
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -184,7 +178,7 @@ async def tus_post(
     kbid: str,
     item: Optional[CreateResourcePayload] = None,
 ) -> Response:
-    return await _tus_post(request, kbid, item=item)
+    return await _tus_post(request, kbid, item)
 
 
 # called by one the three POST above - there are defined distinctly to produce clean API doc
@@ -193,23 +187,29 @@ async def _tus_post(
     kbid: str,
     item: Optional[CreateResourcePayload] = None,
     path_rid: Optional[str] = None,
-    rslug: Optional[str] = None,
-    field: Optional[str] = None,
+    field_id: Optional[str] = None,
 ) -> Response:
     """
     An empty POST request is used to create a new upload resource.
     The Upload-Length header indicates the size of the entire upload in bytes.
     """
+    if path_rid is not None:
+        await validate_rid_exists_or_raise_error(kbid, path_rid)
+
+    kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+    if item and item.hidden and not (kb_config and kb_config.hidden_resources_enabled):
+        raise HTTPException(
+            status_code=422,
+            detail="Cannot hide a resource: the KB does not have hidden resources enabled",
+        )
+
+    await maybe_back_pressure(request, kbid, resource_uuid=path_rid)
+
     dm = get_dm()
     storage_manager = get_storage_manager()
 
-    if rslug is not None:
-        path_rid = await get_rid_from_params_or_raise_error(kbid, slug=rslug)
-
     implies_resource_creation = path_rid is None
 
-    await maybe_back_pressure(request, kbid, resource_uuid=path_rid)
-
     deferred_length = False
     if request.headers.get("upload-defer-length") == "1":
         deferred_length = True
@@ -228,22 +228,11 @@ async def _tus_post(
         try:
             metadata = parse_tus_metadata(request.headers["upload-metadata"])
         except InvalidTUSMetadata as exc:
-            raise HTTPBadRequest(
-                detail=f"Upload-Metadata header contains errors: {str(exc)}"
-            )
+            raise HTTPBadRequest(detail=f"Upload-Metadata header contains errors: {str(exc)}")
     else:
         metadata = {}
 
-    try:
-        path, rid, field = await start_upload_field(
-            kbid, path_rid, field, metadata.get("md5")
-        )
-    except ResourceNotFound:
-        raise HTTPNotFound("Resource is not found or not yet available")
-    except ConflictError:
-        raise HTTPConflict("A resource with the same uploaded file already exists")
-    except IngestNotAvailable:
-        raise HTTPServiceUnavailable("Upload not available right now, try again")
+    path, rid, field = await validate_field_upload(kbid, path_rid, field_id, metadata.get("md5"))
 
     if implies_resource_creation:
         # When uploading a file to a new kb resource, we want to allow multiple
@@ -269,8 +258,15 @@ async def _tus_post(
     request_content_type = None
     if item is None:
         request_content_type = request.headers.get("content-type")
-    if not request_content_type:
-        request_content_type = guess_content_type(metadata["filename"])
+    if request_content_type is None:
+        request_content_type = content_types.guess(metadata["filename"]) or "application/octet-stream"
+
+    if request_content_type is not None and not content_types.valid(request_content_type):
+        raise HTTPException(
+            status_code=415,
+            detail=f"Unsupported content type: {request_content_type}",
+        )
+
     metadata.setdefault("content_type", request_content_type)
 
     metadata["implies_resource_creation"] = implies_resource_creation
@@ -300,9 +296,7 @@ async def _tus_post(
     await dm.save()
 
     # Find the URL for upload, with the same parameter as this call
-    location = api.url_path_for(
-        "Upload information", upload_id=upload_id, **request.path_params
-    )
+    location = api.url_path_for("Upload information", upload_id=upload_id, **request.path_params)
     return Response(
         status_code=201,
         headers={
@@ -319,6 +313,7 @@ async def _tus_post(
     status_code=200,
     openapi_extra={"x-operation-order": 3},
     name="Upload information",
+    summary="Upload information",
 )
 @requires_one([NucliaDBRoles.WRITER])
 @version(1)
@@ -326,7 +321,7 @@ async def tus_head_rslug_prefix(
     request: Request,
     kbid: str,
     rslug: str,
-    field: str,
+    field: FieldIdString,
     upload_id: str,
 ) -> Response:
     return await _tus_head(upload_id)
@@ -338,6 +333,7 @@ async def tus_head_rslug_prefix(
     status_code=200,
     openapi_extra={"x-operation-order": 3},
     name="Upload information",
+    summary="Upload information",
 )
 @requires_one([NucliaDBRoles.WRITER])
 @version(1)
@@ -345,7 +341,7 @@ async def tus_head_rid_prefix(
     request: Request,
     kbid: str,
     path_rid: str,
-    field: str,
+    field: FieldIdString,
     upload_id: str,
 ) -> Response:
     return await _tus_head(upload_id)
@@ -357,6 +353,7 @@ async def tus_head_rid_prefix(
     status_code=200,
     openapi_extra={"x-operation-order": 3},
     name="Upload information",
+    summary="Upload information",
 )
 @requires_one([NucliaDBRoles.WRITER])
 @version(1)
@@ -393,7 +390,7 @@ async def _tus_head(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}/file/{{field}}/{TUSUPLOAD}/{{upload_id}}",
     tags=["Resource field TUS uploads"],
     status_code=200,
-    name="Upload data on a Resource (by slug)",
+    summary="Upload data on a Resource (by slug)",
     openapi_extra={"x-operation-order": 2},
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -402,20 +399,18 @@ async def tus_patch_rslug_prefix(
     request: Request,
     kbid: str,
     rslug: str,
-    field: str,
+    field: FieldIdString,
     upload_id: str,
-    x_synchronous: bool = Header(False),  # type: ignore
 ) -> Response:
-    return await tus_patch(
-        request, kbid, upload_id, rslug=rslug, field=field, x_synchronous=x_synchronous
-    )
+    rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
+    return await tus_patch(request, kbid, upload_id, rid=rid, field=field)
 
 
 @api.patch(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{rid}}/file/{{field}}/{TUSUPLOAD}/{{upload_id}}",
     tags=["Resource field TUS uploads"],
     status_code=200,
-    name="Upload data on a Resource (by id)",
+    summary="Upload data on a Resource (by id)",
     openapi_extra={"x-operation-order": 2},
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -424,20 +419,17 @@ async def tus_patch_rid_prefix(
     request: Request,
     kbid: str,
     rid: str,
-    field: str,
+    field: FieldIdString,
     upload_id: str,
-    x_synchronous: bool = Header(False),  # type: ignore
 ) -> Response:
-    return await tus_patch(
-        request, kbid, upload_id, rid=rid, field=field, x_synchronous=x_synchronous
-    )
+    return await tus_patch(request, kbid, upload_id, rid=rid, field=field)
 
 
 @api.patch(
     f"/{KB_PREFIX}/{{kbid}}/{TUSUPLOAD}/{{upload_id}}",
     tags=["Knowledge Box TUS uploads"],
     status_code=200,
-    name="Upload data on a Knowledge Box",
+    summary="Upload data on a Knowledge Box",
     openapi_extra={"x-operation-order": 2},
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -446,9 +438,8 @@ async def patch(
     request: Request,
     kbid: str,
     upload_id: str,
-    x_synchronous: bool = Header(False),  # type: ignore
 ) -> Response:
-    return await tus_patch(request, kbid, upload_id, x_synchronous=x_synchronous)
+    return await tus_patch(request, kbid, upload_id)
 
 
 async def tus_patch(
@@ -456,9 +447,7 @@ async def tus_patch(
     kbid: str,
     upload_id: str,
     rid: Optional[str] = None,
-    rslug: Optional[str] = None,
     field: Optional[str] = None,
-    x_synchronous: bool = False,
 ):
     try:
         return await _tus_patch(
@@ -466,9 +455,7 @@ async def tus_patch(
             kbid,
             upload_id,
             rid=rid,
-            rslug=rslug,
             field=field,
-            x_synchronous=x_synchronous,
         )
     except ResumableURINotAvailable:
         return HTTPClientError(
@@ -483,15 +470,13 @@ async def _tus_patch(
     kbid: str,
     upload_id: str,
     rid: Optional[str] = None,
-    rslug: Optional[str] = None,
     field: Optional[str] = None,
-    x_synchronous: bool = False,
 ) -> Response:
     """
-    Upload all bytes in the requests and append them in the specifyied offset
+    Upload all bytes in the requests and append them in the specified offset
     """
-    if rslug is not None:
-        rid = await get_rid_from_params_or_raise_error(kbid, slug=rslug)
+    if rid is not None:
+        await validate_rid_exists_or_raise_error(kbid, rid)
 
     dm = get_dm()
     await dm.load(upload_id)
@@ -517,8 +502,7 @@ async def _tus_patch(
 
     if offset != dm.offset:
         raise HTTPConflict(
-            detail=f"Current upload offset({offset}) does not match "
-            f"object offset {dm.offset}"
+            detail=f"Current upload offset({offset}) does not match " f"object offset {dm.offset}"
         )
 
     storage_manager = get_storage_manager()
@@ -530,9 +514,7 @@ async def _tus_patch(
 
     if to_upload and read_bytes != to_upload:  # pragma: no cover
         # check length matches if provided
-        raise HTTPPreconditionFailed(
-            detail="Upload size does not match what was provided"
-        )
+        raise HTTPPreconditionFailed(detail="Upload size does not match what was provided")
     await dm.update(offset=offset + read_bytes)
 
     headers = {
@@ -544,7 +526,6 @@ async def _tus_patch(
     }
 
     upload_finished = dm.get("size") is not None and dm.offset >= dm.get("size")
-
     if upload_finished:
         rid = dm.get("rid", rid)
         if rid is None:
@@ -563,13 +544,19 @@ async def _tus_patch(
             if isinstance(item_payload, str):
                 item_payload = item_payload.encode()
             creation_payload = pickle.loads(base64.b64decode(item_payload))
+
+        content_type = dm.get("metadata", {}).get("content_type")
+        if content_type is not None and not content_types.valid(content_type):
+            return HTTPClientError(
+                status_code=415,
+                detail=f"Unsupported content type: {content_type}",
+            )
+
         try:
             seqid = await store_file_on_nuclia_db(
                 size=dm.get("size"),
-                content_type=dm.get("metadata", {}).get("content_type"),
-                override_resource_title=dm.get("metadata", {}).get(
-                    "implies_resource_creation", False
-                ),
+                content_type=content_type,
+                override_resource_title=dm.get("metadata", {}).get("implies_resource_creation", False),
                 filename=dm.get("metadata", {}).get("filename"),
                 password=dm.get("metadata", {}).get("password"),
                 language=dm.get("metadata", {}).get("language"),
@@ -582,34 +569,30 @@ async def _tus_patch(
                 request=request,
                 bucket=storage_manager.storage.get_bucket_name(kbid),
                 item=creation_payload,
-                wait_on_commit=x_synchronous,
             )
         except LimitsExceededError as exc:
             raise HTTPException(status_code=exc.status_code, detail=exc.detail)
 
         headers["NDB-Seq"] = f"{seqid}"
     else:
-        check_uploaded_chunk_size(read_bytes, storage_manager)
+        validate_intermediate_tus_chunk(read_bytes, storage_manager)
         await dm.save()
 
     return Response(headers=headers)
 
 
-def check_uploaded_chunk_size(read_bytes: int, storage_manager: FileStorageManager):
-    if (
-        storage_manager.min_upload_size is not None
-        and read_bytes < storage_manager.min_upload_size
-    ):
-        raise HTTPPreconditionFailed(
-            detail=f"Intermediate chunks cannot be smaller than {storage_manager.min_upload_size} bytes"
-        )
+def validate_intermediate_tus_chunk(read_bytes: int, storage_manager: FileStorageManager):
+    try:
+        storage_manager.validate_intermediate_chunk(read_bytes)
+    except ValueError as err:
+        raise HTTPPreconditionFailed(detail=str(err))
 
 
 @api.post(
     f"/{KB_PREFIX}/{{kbid}}/{RSLUG_PREFIX}/{{rslug}}/file/{{field}}/{UPLOAD}",
     status_code=201,
     tags=["Resource fields"],
-    name="Upload binary file on a Resource (by slug)",
+    summary="Upload binary file on a Resource (by slug)",
     description="Upload a file as a field on an existing resource, if the field exists will return a conflict (419)",
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -618,23 +601,22 @@ async def upload_rslug_prefix(
     request: StarletteRequest,
     kbid: str,
     rslug: str,
-    field: str,
+    field: FieldIdString,
     x_filename: Optional[list[str]] = Header(None),  # type: ignore
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
-    x_synchronous: bool = Header(False),  # type: ignore
 ) -> ResourceFileUploaded:
+    rid = await get_rid_from_slug_or_raise_error(kbid, rslug)
     return await _upload(
         request,
         kbid,
-        rslug=rslug,
+        path_rid=rid,
         field=field,
         x_filename=x_filename,
         x_password=x_password,
         x_language=x_language,
         x_md5=x_md5,
-        x_synchronous=x_synchronous,
     )
 
 
@@ -642,7 +624,7 @@ async def upload_rslug_prefix(
     f"/{KB_PREFIX}/{{kbid}}/{RESOURCE_PREFIX}/{{path_rid}}/file/{{field}}/{UPLOAD}",
     status_code=201,
     tags=["Resource fields"],
-    name="Upload binary file on a Resource (by id)",
+    summary="Upload binary file on a Resource (by id)",
     description="Upload a file as a field on an existing resource, if the field exists will return a conflict (419)",
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -651,12 +633,11 @@ async def upload_rid_prefix(
     request: StarletteRequest,
     kbid: str,
     path_rid: str,
-    field: str,
+    field: FieldIdString,
     x_filename: Optional[list[str]] = Header(None),  # type: ignore
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
-    x_synchronous: bool = Header(False),  # type: ignore
 ) -> ResourceFileUploaded:
     return await _upload(
         request,
@@ -667,7 +648,6 @@ async def upload_rid_prefix(
         x_password=x_password,
         x_language=x_language,
         x_md5=x_md5,
-        x_synchronous=x_synchronous,
     )
 
 
@@ -675,7 +655,7 @@ async def upload_rid_prefix(
     f"/{KB_PREFIX}/{{kbid}}/{UPLOAD}",
     status_code=201,
     tags=["Knowledge Boxes"],
-    name="Upload binary file on a Knowledge Box",
+    summary="Upload binary file on a Knowledge Box",
     description="Upload a file onto a Knowledge Box, field id will be file and rid will be autogenerated. ",
 )
 @requires_one([NucliaDBRoles.WRITER])
@@ -687,7 +667,6 @@ async def upload(
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
-    x_synchronous: bool = Header(False),  # type: ignore
 ) -> ResourceFileUploaded:
     return await _upload(
         request,
@@ -696,7 +675,6 @@ async def upload(
         x_password=x_password,
         x_language=x_language,
         x_md5=x_md5,
-        x_synchronous=x_synchronous,
     )
 
 
@@ -705,30 +683,19 @@ async def _upload(
     request: StarletteRequest,
     kbid: str,
     path_rid: Optional[str] = None,
-    rslug: Optional[str] = None,
     field: Optional[str] = None,
     x_filename: Optional[list[str]] = Header(None),  # type: ignore
     x_password: Optional[list[str]] = Header(None),  # type: ignore
     x_language: Optional[list[str]] = Header(None),  # type: ignore
     x_md5: Optional[list[str]] = Header(None),  # type: ignore
-    x_synchronous: bool = Header(False),  # type: ignore
 ) -> ResourceFileUploaded:
-    if rslug is not None:
-        path_rid = await get_rid_from_params_or_raise_error(kbid, slug=rslug)
+    if path_rid is not None:
+        await validate_rid_exists_or_raise_error(kbid, path_rid)
 
     await maybe_back_pressure(request, kbid, resource_uuid=path_rid)
 
     md5_user = x_md5[0] if x_md5 is not None and len(x_md5) > 0 else None
-    try:
-        path, rid, valid_field = await start_upload_field(
-            kbid, path_rid, field, md5_user
-        )
-    except ResourceNotFound:
-        raise HTTPNotFound("Resource is not found or not yet available")
-    except ConflictError:
-        raise HTTPConflict("A resource with the same uploaded file already exists")
-    except IngestNotAvailable:
-        raise HTTPServiceUnavailable("Upload not available right now, try again")
+    path, rid, valid_field = await validate_field_upload(kbid, path_rid, field, md5_user)
     dm = get_dm()
     storage_manager = get_storage_manager()
 
@@ -757,8 +724,14 @@ async def _upload(
     # - content-type set by the user in the upload request header takes precedence.
     # - if not set, we will try to guess it from the filename and default to a generic binary content type otherwise
     content_type = request.headers.get("content-type")
-    if not content_type:
-        content_type = guess_content_type(filename)
+    if content_type is None:
+        content_type = content_types.guess(filename) or "application/octet-stream"
+
+    if not content_types.valid(content_type):
+        raise HTTPException(
+            status_code=415,
+            detail=f"Unsupported content type: {content_type}",
+        )
 
     metadata = {"content_type": content_type, "filename": filename}
 
@@ -808,7 +781,6 @@ async def _upload(
             path=path,
             request=request,
             bucket=storage_manager.storage.get_bucket_name(kbid),
-            wait_on_commit=x_synchronous,
         )
     except LimitsExceededError as exc:
         raise HTTPException(status_code=exc.status_code, detail=exc.detail)
@@ -816,43 +788,37 @@ async def _upload(
     return ResourceFileUploaded(seqid=seqid, uuid=rid, field_id=valid_field)
 
 
-async def start_upload_field(
+async def validate_field_upload(
     kbid: str,
     rid: Optional[str] = None,
     field: Optional[str] = None,
     md5: Optional[str] = None,
 ):
-    ingest = get_ingest()
-    pbrequest = ResourceFieldId()
-    pbrequest.kbid = kbid
-    if rid is not None:
-        pbrequest.rid = rid
+    """Validate field upload and return blob storage path, rid and field id.
 
-    elif rid is None and md5 is not None:
-        pbrequest.rid = md5
+    This function assumes KB exists
+    """
 
-    try:
-        response: ResourceFieldExistsResponse = await ingest.ResourceFieldExists(pbrequest)  # type: ignore
-    except AioRpcError as exc:
-        if exc.code() is GrpcStatusCode.UNAVAILABLE:
-            raise IngestNotAvailable()
+    if rid is None:
+        # we are going to create a new resource and a field
+        if md5 is not None:
+            exists = await datamanagers.atomic.resources.resource_exists(kbid=kbid, rid=md5)
+            if exists:
+                raise HTTPConflict("A resource with the same uploaded file already exists")
+            rid = md5
         else:
-            raise exc
-
-    if response.found is False and rid is not None:
-        raise ResourceNotFound()
-    elif response.found is True and rid is None and md5 is not None:
-        raise ConflictError()
-
-    if rid is None and md5 is None:
-        rid = uuid.uuid4().hex
-    elif rid is None:
-        rid = md5
-
-    if field is None and md5 is None:
-        field = uuid.uuid4().hex
-    elif field is None:
-        field = md5
+            rid = uuid.uuid4().hex
+    else:
+        # we're adding a field to a resource
+        exists = await datamanagers.atomic.resources.resource_exists(kbid=kbid, rid=rid)
+        if not exists:
+            raise HTTPNotFound("Resource is not found or not yet available")
+
+    if field is None:
+        if md5 is None:
+            field = uuid.uuid4().hex
+        else:
+            field = md5
 
     path = KB_RESOURCE_FIELD.format(kbid=kbid, uuid=rid, field=field)
     return path, rid, field
@@ -864,7 +830,7 @@ async def store_file_on_nuclia_db(
     path: str,
     request: Request,
     bucket: str,
-    source: Source,
+    source: CloudFile.Source.ValueType,
     rid: str,
     field: str,
     content_type: str = "application/octet-stream",
@@ -874,12 +840,9 @@ async def store_file_on_nuclia_db(
     language: Optional[str] = None,
     md5: Optional[str] = None,
     item: Optional[CreateResourcePayload] = None,
-    wait_on_commit: bool = False,
 ) -> Optional[int]:
     # File is on NucliaDB Storage at path
-
     partitioning = get_partitioning()
-    transaction = get_transaction_utility()
     processing = get_processing()
     storage = await get_storage(service_name=SERVICE_NAME)
 
@@ -901,14 +864,17 @@ async def store_file_on_nuclia_db(
 
     parse_audit(writer.audit, request)
 
+    unique_slug_context_manager = noop_context_manager()
     if item is not None:
         if item.slug:
+            unique_slug_context_manager = ensure_slug_uniqueness(kbid, item.slug)
             writer.slug = item.slug
             toprocess.slug = item.slug
 
         toprocess.processing_options = item.processing_options
 
-        parse_basic(writer, item, toprocess)
+        kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+        parse_basic_creation(writer, item, toprocess, kb_config)
         if item.origin is not None:
             parse_origin(writer.origin, item.origin)
         if item.extra is not None:
@@ -924,52 +890,61 @@ async def store_file_on_nuclia_db(
             uuid=rid,
             x_skip_store=False,
         )
+    else:
+        # Use defaults for everything, but don't forget hidden which depends on KB config
+        kb_config = await datamanagers.atomic.kb.get_config(kbid=kbid)
+        if kb_config and kb_config.hidden_resources_hide_on_creation:
+            writer.basic.hidden = True
+
+    async with unique_slug_context_manager:
+        if override_resource_title and filename is not None:
+            set_title(writer, toprocess, filename)
+
+        writer.basic.icon = content_type
+        writer.basic.created.FromDatetime(datetime.now())
+
+        # Update resource with file
+        file_field = FieldFile()
+        file_field.added.FromDatetime(datetime.now())
+        file_field.file.bucket_name = bucket
+        file_field.file.content_type = content_type
+        if filename is not None:
+            file_field.file.filename = filename
+        file_field.file.uri = path
+        file_field.file.source = source
+
+        if md5:
+            file_field.file.md5 = md5
+        if size:
+            file_field.file.size = size
+        if language:
+            file_field.language = language
+        if password:
+            file_field.password = password
+
+        writer.files[field].CopyFrom(file_field)
+        # Do not store passwords on maindb
+        writer.files[field].ClearField("password")
+
+        toprocess.filefield[field] = await processing.convert_internal_filefield_to_str(
+            file_field, storage=storage
+        )
 
-    if override_resource_title and filename is not None:
-        set_title(writer, toprocess, filename)
-
-    writer.basic.icon = content_type
-    writer.basic.created.FromDatetime(datetime.now())
-
-    # Update resource with file
-    file_field = FieldFile()
-    file_field.added.FromDatetime(datetime.now())
-    file_field.file.bucket_name = bucket
-    file_field.file.content_type = content_type
-    if filename is not None:
-        file_field.file.filename = filename
-    file_field.file.uri = path
-    file_field.file.source = source
-
-    if md5:
-        file_field.file.md5 = md5
-    if size:
-        file_field.file.size = size
-    if language:
-        file_field.language = language
-    if password:
-        file_field.password = password
-
-    writer.files[field].CopyFrom(file_field)
-    # Do not store passwords on maindb
-    writer.files[field].ClearField("password")
-
-    toprocess.filefield[field] = await processing.convert_internal_filefield_to_str(
-        file_field, storage=storage
-    )
-
-    try:
-        processing_info = await processing.send_to_process(toprocess, partition)
-    except LimitsExceededError as exc:
-        raise HTTPException(status_code=exc.status_code, detail=exc.detail)
-    except SendToProcessError:
-        raise HTTPException(status_code=500, detail="Error while sending to process")
-
-    writer.source = BrokerMessage.MessageSource.WRITER
-    set_processing_info(writer, processing_info)
-    await transaction.commit(writer, partition, wait=wait_on_commit)
+        writer.source = BrokerMessage.MessageSource.WRITER
+        writer.basic.metadata.status = Metadata.Status.PENDING
+        writer.basic.metadata.useful = True
+        await transaction.commit(writer, partition)
+        try:
+            processing_info = await processing.send_to_process(toprocess, partition)
+        except LimitsExceededError as exc:
+            raise HTTPException(status_code=exc.status_code, detail=exc.detail)
+        except SendToProcessError:
+            raise HTTPException(
+                status_code=500,
+                detail="Error while sending to process. Try calling /reprocess",
+            )
 
-    return processing_info.seqid
+        return processing_info.seqid
 
 
 def maybe_b64decode(some_string: str) -> str:
@@ -978,9 +953,3 @@ def maybe_b64decode(some_string: str) -> str:
     except ValueError:
         # not b64encoded
         return some_string
-
-
-def guess_content_type(filename: str) -> str:
-    default = "application/octet-stream"
-    guessed, _ = mimetypes.guess_type(filename)
-    return guessed or default