nucliadb 6.7.2.post4874__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

nucliadb/export_import/utils.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import functools
-from typing import AsyncGenerator, AsyncIterator, Callable, Optional
+from collections.abc import AsyncGenerator, AsyncIterator, Callable
 
 import backoff
 from google.protobuf.message import DecodeError as ProtobufDecodeError
@@ -35,6 +35,7 @@ from nucliadb.export_import.exceptions import (
 )
 from nucliadb.export_import.models import ExportedItemType, ExportItem, Metadata
 from nucliadb.ingest.orm.broker_message import generate_broker_message
+from nucliadb.ingest.orm.resource import Resource
 from nucliadb_models.configuration import SearchConfiguration
 from nucliadb_models.export_import import Status
 from nucliadb_protos import knowledgebox_pb2 as kb_pb2
@@ -171,14 +172,6 @@ async def import_binary(
     )
 
 
-async def set_entities_groups(
-    context: ApplicationContext, kbid: str, entities_groups: kb_pb2.EntitiesGroups
-) -> None:
-    async with datamanagers.with_transaction() as txn:
-        await datamanagers.entities.set_entities_groups(txn, kbid=kbid, entities_groups=entities_groups)
-        await txn.commit()
-
-
 async def set_synonyms(context: ApplicationContext, kbid: str, synonyms: kb_pb2.Synonyms) -> None:
     async with datamanagers.with_transaction() as txn:
         await datamanagers.synonyms.set(txn, kbid=kbid, synonyms=synonyms)
@@ -207,9 +200,9 @@ async def iter_kb_resource_uuids(context: ApplicationContext, kbid: str) -> Asyn
 
 async def get_broker_message(
     context: ApplicationContext, kbid: str, rid: str
-) -> Optional[writer_pb2.BrokerMessage]:
+) -> writer_pb2.BrokerMessage | None:
     async with datamanagers.with_ro_transaction() as txn:
-        resource = await datamanagers.resources.get_resource(txn, kbid=kbid, rid=rid)
+        resource = await Resource.get(txn, kbid=kbid, rid=rid)
         if resource is None:
             return None
         resource.disable_vectors = False
@@ -284,11 +277,6 @@ async def download_binary(
     assert downloaded_bytes == cf.size, "Downloaded bytes do not match the expected size"
 
 
-async def get_entities(context: ApplicationContext, kbid: str) -> kb_pb2.EntitiesGroups:
-    async with datamanagers.with_ro_transaction() as txn:
-        return await datamanagers.entities.get_entities_groups(txn, kbid=kbid)
-
-
 async def get_labels(context: ApplicationContext, kbid: str) -> kb_pb2.Labels:
     async with datamanagers.with_ro_transaction() as txn:
         return await datamanagers.labels.get_labels(txn, kbid=kbid)
@@ -434,7 +422,7 @@ class ExportStreamReader:
 
     async def maybe_read_learning_config(
         self,
-    ) -> tuple[Optional[learning_proxy.LearningConfiguration], bytes]:
+    ) -> tuple[learning_proxy.LearningConfiguration | None, bytes]:
         """
         Tries to read a learning config from the beginning of the stream.
         Returs the learning config if found. It also returns any leftover bytes that
@@ -533,7 +521,7 @@ class TaskRetryHandler:
 
 async def get_learning_config(
     kbid: str,
-) -> Optional[learning_proxy.LearningConfiguration]:
+) -> learning_proxy.LearningConfiguration | None:
     return await learning_proxy.get_configuration(kbid)
 
 

nucliadb/health.py

@@ -19,7 +19,7 @@
 #
 import asyncio
 import logging
-from typing import Awaitable, Callable, Optional
+from collections.abc import Awaitable, Callable
 
 from grpc import aio
 from grpc_health.v1 import health, health_pb2, health_pb2_grpc
@@ -41,7 +41,7 @@ def nats_manager_healthy() -> bool:
 
 
 def pubsub_check() -> bool:
-    driver: Optional[PubSubDriver] = get_utility(Utility.PUBSUB)
+    driver: PubSubDriver | None = get_utility(Utility.PUBSUB)
     if driver is None:
         return True
     if isinstance(driver, NatsPubsub):

nucliadb/ingest/app.py

@@ -19,7 +19,7 @@
 #
 import asyncio
 import importlib.metadata
-from typing import Awaitable, Callable
+from collections.abc import Awaitable, Callable
 
 from nucliadb import health
 from nucliadb.backups.tasks import initialize_consumers as initialize_backup_consumers
@@ -96,7 +96,7 @@ async def initialize_grpc():  # pragma: no cover
     finalizers = await initialize()
     grpc_finalizer = await start_grpc(SERVICE_NAME)
 
-    return [grpc_finalizer] + finalizers
+    return [grpc_finalizer, *finalizers]
 
 
 async def initialize_pull_workers() -> list[Callable[[], Awaitable[None]]]:
@@ -114,14 +114,14 @@ async def main_consumer():  # pragma: no cover
 
     ingest_consumers = await consumer_service.start_ingest_consumers(SERVICE_NAME)
 
-    await run_until_exit([grpc_health_finalizer, ingest_consumers, metrics_server.shutdown] + finalizers)
+    await run_until_exit([grpc_health_finalizer, ingest_consumers, metrics_server.shutdown, *finalizers])
 
 
 async def main_orm_grpc():  # pragma: no cover
     finalizers = await initialize()
     grpc_finalizer = await start_grpc(SERVICE_NAME)
     metrics_server = await serve_metrics()
-    await run_until_exit([grpc_finalizer, metrics_server.shutdown] + finalizers)
+    await run_until_exit([grpc_finalizer, metrics_server.shutdown, *finalizers])
 
 
 async def main_ingest_processed_consumer():  # pragma: no cover
@@ -134,7 +134,7 @@ async def main_ingest_processed_consumer():  # pragma: no cover
     consumer = await consumer_service.start_ingest_processed_consumer_v2(SERVICE_NAME)
 
     await run_until_exit(
-        [grpc_health_finalizer, consumer, metrics_server.shutdown, stop_processing_engine] + finalizers
+        [grpc_health_finalizer, consumer, metrics_server.shutdown, stop_processing_engine, *finalizers]
     )
 
 
@@ -158,8 +158,8 @@ async def main_subscriber_workers():  # pragma: no cover
     backup_consumers_finalizers = await initialize_backup_consumers(context)
 
     await run_until_exit(
-        backup_consumers_finalizers
-        + [
+        [
+            *backup_consumers_finalizers,
             imports_consumer.finalize,
             exports_consumer.finalize,
             stop_ingest_utility,
@@ -169,8 +169,8 @@ async def main_subscriber_workers():  # pragma: no cover
             grpc_health_finalizer,
             metrics_server.shutdown,
             context.finalize,
+            *finalizers,
         ]
-        + finalizers
     )
 
 

nucliadb/ingest/consumer/consumer.py

@@ -20,12 +20,10 @@
 import asyncio
 import logging
 import time
-from typing import Optional, Union
 
 import backoff
 import nats
 import nats.js.api
-import nats.js.errors
 from nats.aio.client import Msg
 from nats.js import JetStreamContext
 
@@ -74,8 +72,8 @@ class IngestConsumer:
         partition: str,
         storage: Storage,
         nats_connection_manager: NatsConnectionManager,
-        pubsub: Optional[PubSubDriver] = None,
-        lock: Optional[Union[asyncio.Lock, asyncio.Semaphore]] = None,
+        pubsub: PubSubDriver | None = None,
+        lock: asyncio.Lock | asyncio.Semaphore | None = None,
     ):
         self.driver = driver
         self.partition = partition
@@ -85,9 +83,9 @@ class IngestConsumer:
 
         self.lock = lock or asyncio.Lock()
         self.processor = Processor(driver, storage, pubsub, partition)
-        self.subscription: Optional[JetStreamContext.PullSubscription] = None
+        self.subscription: JetStreamContext.PullSubscription | None = None
 
-    async def ack_message(self, msg: Msg, kbid: Optional[str] = None):
+    async def ack_message(self, msg: Msg, kbid: str | None = None):
         await msg.ack()
 
     async def initialize(self):
@@ -162,7 +160,7 @@ class IngestConsumer:
     async def subscription_worker(self, msg: Msg):
         context.clear_context()
 
-        kbid: Optional[str] = None
+        kbid: str | None = None
         subject = msg.subject
         reply = msg.reply
         seqid = int(reply.split(".")[5])
@@ -238,7 +236,7 @@ class IngestConsumer:
                 logger.info(
                     f"An error happend while processing a message from {message_source}. "
                     f"A copy of the message has been stored on {self.processor.storage.deadletter_bucket}. "
-                    f"Check sentry for more details: {str(e)}"
+                    f"Check sentry for more details: {e!s}"
                 )
                 await self.ack_message(msg, kbid)
                 logger.info("Message acked because of deadletter", extra={"seqid": seqid})
@@ -250,7 +248,7 @@ class IngestConsumer:
                 logger.info(
                     f"An error happend while processing a message from {message_source}. "
                     f"This message has been dropped and won't be retried again"
-                    f"Check sentry for more details: {str(e)}"
+                    f"Check sentry for more details: {e!s}"
                 )
                 await self.ack_message(msg, kbid)
                 logger.info("Message acked because of drop", extra={"seqid": seqid})
@@ -260,7 +258,7 @@ class IngestConsumer:
                 logger.exception(
                     f"An error happend while processing a message from {message_source}. "
                     "Message has not been ACKd and will be retried. "
-                    f"Check sentry for more details: {str(e)}"
+                    f"Check sentry for more details: {e!s}"
                 )
                 await msg.nak()
                 logger.info("Message nacked because of unhandled error", extra={"seqid": seqid})

nucliadb/ingest/consumer/pull.py

@@ -21,7 +21,6 @@ import asyncio
 import base64
 import time
 from contextlib import contextmanager
-from typing import Optional
 
 from aiohttp.client_exceptions import ClientConnectorError
 from opentelemetry import trace
@@ -31,10 +30,10 @@ from opentelemetry.trace import (
     Link,
 )
 
+from nucliadb.common.http_clients.exceptions import ServiceUnavailableException
 from nucliadb.common.http_clients.processing import (
     ProcessingHTTPClient,
     ProcessingPullMessageProgressUpdater,
-    get_nua_api_id,
 )
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.ingest import SERVICE_NAME, logger, logger_activity
@@ -95,7 +94,7 @@ class PullV2Worker:
         driver: Driver,
         storage: Storage,
         pull_time_error_backoff: int,
-        pubsub: Optional[PubSubDriver] = None,
+        pubsub: PubSubDriver | None = None,
         pull_time_empty_backoff: float = 5.0,
         pull_api_timeout: int = 60,
     ):
@@ -141,12 +140,9 @@ class PullV2Worker:
         data = None
         if nuclia_settings.nuclia_service_account is not None:
             headers["X-STF-NUAKEY"] = f"Bearer {nuclia_settings.nuclia_service_account}"
-            # parse jwt sub to get pull type id
-            try:
-                get_nua_api_id()
-            except Exception as exc:
+            if nuclia_settings.nuclia_service_account is None:
                 logger.exception("Could not read NUA API Key. Can not start pull worker")
-                raise ReallyStopPulling() from exc
+                raise ReallyStopPulling()
 
         ack_tokens = []
         async with ProcessingHTTPClient() as processing_http_client:
@@ -209,6 +205,12 @@ class PullV2Worker:
                             payload_length = len(base64.b64decode(data.payload))
                         logger.error(f"Message too big for transaction: {payload_length}")
                     raise e
+
+                except ServiceUnavailableException as ex:
+                    logger.warning(f"Processing api is unavailable, will retry shortly: {ex}")
+                    await processing_http_client.reset_session()
+                    await asyncio.sleep(self.pull_time_error_backoff)
+
                 except Exception:
                     logger.exception("Unhandled error pulling messages from processing")
                     await asyncio.sleep(self.pull_time_error_backoff)

nucliadb/ingest/consumer/service.py

@@ -19,24 +19,21 @@
 #
 import asyncio
 import sys
+from collections.abc import Awaitable, Callable
 from functools import partial
-from typing import Awaitable, Callable, Optional
 
-from nucliadb.common.back_pressure.materializer import BackPressureMaterializer
-from nucliadb.common.back_pressure.settings import settings as back_pressure_settings
 from nucliadb.common.maindb.utils import setup_driver
 from nucliadb.ingest import SERVICE_NAME, logger
 from nucliadb.ingest.consumer.consumer import IngestConsumer
 from nucliadb.ingest.consumer.pull import PullV2Worker
 from nucliadb.ingest.settings import settings
 from nucliadb_utils.exceptions import ConfigurationError
-from nucliadb_utils.settings import indexing_settings, transaction_settings
+from nucliadb_utils.settings import transaction_settings
 from nucliadb_utils.utilities import (
     get_audit,
     get_nats_manager,
     get_pubsub,
     get_storage,
-    start_nats_manager,
 )
 
 from .auditing import IndexAuditHandler, ResourceWritesAuditHandler
@@ -57,29 +54,8 @@ async def _exit_tasks(tasks: list[asyncio.Task]) -> None:
     await asyncio.gather(*tasks, return_exceptions=True)
 
 
-async def start_back_pressure() -> BackPressureMaterializer:
-    logger.info("Starting back pressure materializer")
-    nats_manager = await start_nats_manager(
-        SERVICE_NAME,
-        indexing_settings.index_jetstream_servers,
-        indexing_settings.index_jetstream_auth,
-    )
-    back_pressure = BackPressureMaterializer(
-        nats_manager,
-        indexing_check_interval=back_pressure_settings.indexing_check_interval,
-        ingest_check_interval=back_pressure_settings.ingest_check_interval,
-    )
-    await back_pressure.start()
-    return back_pressure
-
-
-async def stop_back_pressure(materializer: BackPressureMaterializer) -> None:
-    await materializer.stop()
-    await materializer.nats_manager.finalize()
-
-
 async def start_ingest_consumers(
-    service_name: Optional[str] = None,
+    service_name: str | None = None,
 ) -> Callable[[], Awaitable[None]]:
     if transaction_settings.transaction_local:
         raise ConfigurationError("Can not start ingest consumers in local mode")
@@ -115,7 +91,7 @@ async def start_ingest_consumers(
 
 
 async def start_ingest_processed_consumer_v2(
-    service_name: Optional[str] = None,
+    service_name: str | None = None,
 ) -> Callable[[], Awaitable[None]]:
     """
     This is not meant to be deployed with a stateful set like the other consumers.
@@ -164,9 +140,8 @@ async def start_shard_creator() -> Callable[[], Awaitable[None]]:
     driver = await setup_driver()
     pubsub = await get_pubsub()
     assert pubsub is not None, "Pubsub is not configured"
-    storage = await get_storage(service_name=SERVICE_NAME)
 
-    shard_creator = ShardCreatorHandler(driver=driver, storage=storage, pubsub=pubsub)
+    shard_creator = ShardCreatorHandler(driver=driver, pubsub=pubsub)
     await shard_creator.initialize()
 
     return shard_creator.finalize

nucliadb/ingest/consumer/shard_creator.py

@@ -25,14 +25,14 @@ from typing import Any
 
 from nidx_protos import nodereader_pb2, noderesources_pb2
 
-from nucliadb.common import locking
+from nucliadb.common import datamanagers, locking
+from nucliadb.common.cluster.settings import settings
 from nucliadb.common.cluster.utils import get_shard_manager
 from nucliadb.common.maindb.driver import Driver
 from nucliadb.common.nidx import get_nidx_api_client
 from nucliadb_protos import writer_pb2
 from nucliadb_utils import const
 from nucliadb_utils.cache.pubsub import PubSubDriver
-from nucliadb_utils.storages.storage import Storage
 
 from . import metrics
 from .utils import DelayedTaskHandler
@@ -52,12 +52,10 @@ class ShardCreatorHandler:
         self,
         *,
         driver: Driver,
-        storage: Storage,
         pubsub: PubSubDriver,
         check_delay: float = 10.0,
     ):
         self.driver = driver
-        self.storage = storage
         self.pubsub = pubsub
         self.shard_manager = get_shard_manager()
         self.task_handler = DelayedTaskHandler(check_delay)
@@ -111,4 +109,17 @@ class ShardCreatorHandler:
                     shard_id=noderesources_pb2.ShardId(id=current_shard.nidx_shard_id)
                 )  # type: ignore
             )
-            await self.shard_manager.maybe_create_new_shard(kbid, shard.paragraphs)
+
+            if not should_create_new_shard(shard.paragraphs):
+                return
+
+            logger.info({"message": "Adding shard", "kbid": kbid})
+            async with datamanagers.with_rw_transaction() as txn:
+                kb_config = await datamanagers.kb.get_config(txn, kbid=kbid)
+                prewarm = kb_config is not None and kb_config.prewarm_enabled
+                await self.shard_manager.create_shard_by_kbid(txn, kbid, prewarm_enabled=prewarm)
+                await txn.commit()
+
+
+def should_create_new_shard(num_paragraphs: int) -> bool:
+    return num_paragraphs > settings.max_shard_paragraphs

nucliadb/ingest/consumer/utils.py

@@ -18,7 +18,7 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import asyncio
-from typing import Callable, Coroutine
+from collections.abc import Callable, Coroutine
 
 
 class DelayedTaskHandler:

nucliadb/ingest/fields/base.py

@@ -24,11 +24,12 @@ import enum
 import logging
 from collections import defaultdict
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Generic, Optional, Type, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 from google.protobuf.message import DecodeError, Message
 
 from nucliadb.common import datamanagers
+from nucliadb.common.ids import FieldId
 from nucliadb.ingest.fields.exceptions import InvalidFieldClass, InvalidPBClass
 from nucliadb_protos.knowledgebox_pb2 import VectorSetConfig
 from nucliadb_protos.resources_pb2 import (
@@ -46,10 +47,8 @@ from nucliadb_protos.resources_pb2 import (
 )
 from nucliadb_protos.utils_pb2 import ExtractedText, VectorObject
 from nucliadb_protos.writer_pb2 import Error, FieldStatus
-from nucliadb_utils import const
 from nucliadb_utils.storages.exceptions import CouldNotCopyNotFound
 from nucliadb_utils.storages.storage import Storage, StorageField
-from nucliadb_utils.utilities import has_feature
 
 logger = logging.getLogger(__name__)
 
@@ -76,27 +75,27 @@ PbType = TypeVar("PbType", bound=Message)
 
 
 class Field(Generic[PbType]):
-    pbklass: Type[PbType]
+    pbklass: type[PbType]
     type: str = "x"
-    value: Optional[Any]
-    extracted_text: Optional[ExtractedText]
-    extracted_vectors: dict[Optional[str], VectorObject]
-    computed_metadata: Optional[FieldComputedMetadata]
-    large_computed_metadata: Optional[LargeComputedMetadata]
-    question_answers: Optional[FieldQuestionAnswers]
+    value: Any | None
+    extracted_text: ExtractedText | None
+    extracted_vectors: dict[str | None, VectorObject]
+    computed_metadata: FieldComputedMetadata | None
+    large_computed_metadata: LargeComputedMetadata | None
+    question_answers: FieldQuestionAnswers | None
 
     def __init__(
         self,
         id: str,
         resource: Resource,
-        pb: Optional[Any] = None,
-        value: Optional[Any] = None,
+        pb: Any | None = None,
+        value: Any | None = None,
     ):
         if self.pbklass is None:
             raise InvalidFieldClass()
 
         self.value = None
-        self.extracted_text: Optional[ExtractedText] = None
+        self.extracted_text: ExtractedText | None = None
         self.extracted_vectors = {}
         self.computed_metadata = None
         self.large_computed_metadata = None
@@ -119,12 +118,20 @@ class Field(Generic[PbType]):
 
     @property
     def kbid(self) -> str:
-        return self.resource.kb.kbid
+        return self.resource.kbid
 
     @property
     def uuid(self) -> str:
         return self.resource.uuid
 
+    @property
+    def field_id(self) -> FieldId:
+        return FieldId(
+            rid=self.resource.uuid,
+            type=self.type,
+            key=self.id,
+        )
+
     @property
     def storage(self) -> Storage:
         return self.resource.storage
@@ -152,7 +159,7 @@ class Field(Generic[PbType]):
 
         return self.storage.file_extracted(self.kbid, self.uuid, self.type, self.id, key)
 
-    async def db_get_value(self) -> Optional[PbType]:
+    async def db_get_value(self) -> PbType | None:
         if self.value is None:
             payload = await datamanagers.fields.get_raw(
                 self.resource.txn,
@@ -215,21 +222,6 @@ class Field(Generic[PbType]):
     ) -> None:
         # Try delete vectors
         sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
-
-        if has_feature(const.Features.DEBUG_MISSING_VECTORS):
-            # This is a very chatty log. It is just a temporary hint while debugging an issue.
-            logger.info(
-                "Deleting vectors from storage",
-                extra={
-                    "kbid": self.kbid,
-                    "rid": self.resource.uuid,
-                    "field": f"{self.type}/{self.id}",
-                    "vectorset": vectorset,
-                    "storage_key_kind": storage_key_kind,
-                    "key": sf.key,
-                    "bucket": sf.bucket,
-                },
-            )
         try:
             await self.storage.delete_upload(sf.key, sf.bucket)
         except KeyError:
@@ -242,7 +234,7 @@ class Field(Generic[PbType]):
         except KeyError:
             pass
 
-    async def get_error(self) -> Optional[Error]:
+    async def get_error(self) -> Error | None:
         return await datamanagers.fields.get_error(
             self.resource.txn,
             kbid=self.kbid,
@@ -261,7 +253,7 @@ class Field(Generic[PbType]):
             error=error,
         )
 
-    async def get_status(self) -> Optional[FieldStatus]:
+    async def get_status(self) -> FieldStatus | None:
         return await datamanagers.fields.get_status(
             self.resource.txn,
             kbid=self.kbid,
@@ -280,7 +272,7 @@ class Field(Generic[PbType]):
             status=status,
         )
 
-    async def get_question_answers(self, force=False) -> Optional[FieldQuestionAnswers]:
+    async def get_question_answers(self, force=False) -> FieldQuestionAnswers | None:
         if self.question_answers is None or force:
             sf = self.get_storage_field(FieldTypes.QUESTION_ANSWERS)
             try:
@@ -297,9 +289,7 @@ class Field(Generic[PbType]):
     async def set_question_answers(self, payload: FieldQuestionAnswerWrapper) -> None:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[FieldQuestionAnswers] = await self.get_question_answers(
-                    force=True
-                )
+                actual_payload: FieldQuestionAnswers | None = await self.get_question_answers(force=True)
             except KeyError:
                 actual_payload = None
         else:
@@ -332,7 +322,7 @@ class Field(Generic[PbType]):
             self.question_answers = actual_payload
 
     async def set_extracted_text(self, payload: ExtractedTextWrapper) -> None:
-        actual_payload: Optional[ExtractedText] = None
+        actual_payload: ExtractedText | None = None
         if self.type in SUBFIELDFIELDS:
             # Try to get the previously extracted text protobuf if it exists so we can merge it with the new splits
             # coming from the processing payload.
@@ -383,7 +373,7 @@ class Field(Generic[PbType]):
             await self.storage.upload_pb(sf, actual_payload)
             self.extracted_text = actual_payload
 
-    async def get_extracted_text(self, force=False) -> Optional[ExtractedText]:
+    async def get_extracted_text(self, force=False) -> ExtractedText | None:
         if self.extracted_text is None or force:
             async with self.locks["extracted_text"]:
                 # Value could have been fetched while waiting for the lock
@@ -399,10 +389,10 @@ class Field(Generic[PbType]):
         payload: ExtractedVectorsWrapper,
         vectorset: str,
         storage_key_kind: VectorSetConfig.StorageKeyKind.ValueType,
-    ) -> Optional[VectorObject]:
+    ) -> VectorObject | None:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[VectorObject] = await self.get_vectors(
+                actual_payload: VectorObject | None = await self.get_vectors(
                     vectorset=vectorset,
                     storage_key_kind=storage_key_kind,
                     force=True,
@@ -413,7 +403,7 @@ class Field(Generic[PbType]):
             actual_payload = None
 
         sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
-        vo: Optional[VectorObject] = None
+        vo: VectorObject | None = None
         if actual_payload is None:
             # Its first extracted vectors
             if payload.HasField("file"):
@@ -465,7 +455,7 @@ class Field(Generic[PbType]):
         vectorset: str,
         storage_key_kind: VectorSetConfig.StorageKeyKind.ValueType,
         force: bool = False,
-    ) -> Optional[VectorObject]:
+    ) -> VectorObject | None:
         if self.extracted_vectors.get(vectorset, None) is None or force:
             sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
             payload = await self.storage.download_pb(sf, VectorObject)
@@ -476,9 +466,7 @@ class Field(Generic[PbType]):
     async def set_field_metadata(self, payload: FieldComputedMetadataWrapper) -> FieldComputedMetadata:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[FieldComputedMetadata] = await self.get_field_metadata(
-                    force=True
-                )
+                actual_payload: FieldComputedMetadata | None = await self.get_field_metadata(force=True)
             except KeyError:
                 actual_payload = None
         else:
@@ -521,7 +509,7 @@ class Field(Generic[PbType]):
 
         return self.computed_metadata
 
-    async def get_field_metadata(self, force: bool = False) -> Optional[FieldComputedMetadata]:
+    async def get_field_metadata(self, force: bool = False) -> FieldComputedMetadata | None:
         if self.computed_metadata is None or force:
             async with self.locks["field_metadata"]:
                 # Value could have been fetched while waiting for the lock
@@ -535,7 +523,7 @@ class Field(Generic[PbType]):
     async def set_large_field_metadata(self, payload: LargeComputedMetadataWrapper):
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[LargeComputedMetadata] = await self.get_large_field_metadata(
+                actual_payload: LargeComputedMetadata | None = await self.get_large_field_metadata(
                     force=True
                 )
             except KeyError:
@@ -545,7 +533,7 @@ class Field(Generic[PbType]):
 
         sf = self.get_storage_field(FieldTypes.FIELD_LARGE_METADATA)
 
-        new_payload: Optional[LargeComputedMetadata] = None
+        new_payload: LargeComputedMetadata | None = None
         if payload.HasField("file"):
             new_payload = LargeComputedMetadata()
             data = await self.storage.downloadbytescf(payload.file)
@@ -572,7 +560,7 @@ class Field(Generic[PbType]):
 
         return self.large_computed_metadata
 
-    async def get_large_field_metadata(self, force: bool = False) -> Optional[LargeComputedMetadata]:
+    async def get_large_field_metadata(self, force: bool = False) -> LargeComputedMetadata | None:
         if self.large_computed_metadata is None or force:
             sf = self.get_storage_field(FieldTypes.FIELD_LARGE_METADATA)
             payload = await self.storage.download_pb(