nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/ingest/consumer/service.py

@@ -45,9 +45,7 @@ from .shard_creator import ShardCreatorHandler
 def _handle_task_result(task: asyncio.Task) -> None:
     e = task.exception()
     if e:
-        logger.exception(
-            "Loop stopped by exception. This should not happen. Exiting.", exc_info=e
-        )
+        logger.exception("Loop stopped by exception. This should not happen. Exiting.", exc_info=e)
         sys.exit(1)
 
 
@@ -87,9 +85,7 @@ async def start_ingest_consumers(
     if transaction_settings.transaction_local:
         raise ConfigurationError("Can not start ingest consumers in local mode")
 
-    while len(
-        manager.get_index_nodes()
-    ) == 0 and running_settings.running_environment not in (
+    while len(manager.get_index_nodes()) == 0 and running_settings.running_environment not in (
         "local",
         "test",
     ):
@@ -101,9 +97,9 @@ async def start_ingest_consumers(
     storage = await get_storage(service_name=service_name or SERVICE_NAME)
     nats_connection_manager = get_nats_manager()
 
-    max_concurrent_processing = asyncio.Semaphore(
-        settings.max_concurrent_ingest_processing
-    )
+    max_concurrent_processing = asyncio.Semaphore(settings.max_concurrent_ingest_processing)
+
+    consumer_finalizers = []
 
     for partition in settings.partitions:
         consumer = IngestConsumer(
@@ -115,8 +111,15 @@ async def start_ingest_consumers(
             lock=max_concurrent_processing,
         )
         await consumer.initialize()
+        consumer_finalizers.append(consumer.finalize)
 
-    return nats_connection_manager.finalize
+    async def _finalize():
+        # Finalize all the consumers and the nats connection manager
+        for consumer_finalize in consumer_finalizers:
+            await consumer_finalize()
+        await nats_connection_manager.finalize()
+
+    return _finalize
 
 
 async def start_ingest_processed_consumer(
@@ -132,9 +135,7 @@ async def start_ingest_processed_consumer(
     if transaction_settings.transaction_local:
         raise ConfigurationError("Can not start ingest consumers in local mode")
 
-    while len(
-        manager.get_index_nodes()
-    ) == 0 and running_settings.running_environment not in (
+    while len(manager.get_index_nodes()) == 0 and running_settings.running_environment not in (
         "local",
         "test",
     ):
@@ -161,19 +162,20 @@ async def start_ingest_processed_consumer(
 async def start_auditor() -> Callable[[], Awaitable[None]]:
     audit = get_audit()
     assert audit is not None
+
     pubsub = await get_pubsub()
     assert pubsub is not None, "Pubsub is not configured"
     storage = await get_storage(service_name=SERVICE_NAME)
     index_auditor = IndexAuditHandler(audit=audit, pubsub=pubsub)
-    resource_writes_auditor = ResourceWritesAuditHandler(
-        storage=storage, audit=audit, pubsub=pubsub
-    )
+    resource_writes_auditor = ResourceWritesAuditHandler(storage=storage, audit=audit, pubsub=pubsub)
 
     await index_auditor.initialize()
     await resource_writes_auditor.initialize()
 
     return partial(
-        asyncio.gather, index_auditor.finalize(), resource_writes_auditor.finalize()  # type: ignore
+        asyncio.gather,
+        index_auditor.finalize(),
+        resource_writes_auditor.finalize(),  # type: ignore
     )
 
 

nucliadb/ingest/consumer/shard_creator.py

@@ -82,9 +82,7 @@ class ShardCreatorHandler:
             metrics.total_messages.inc({"type": "shard_creator", "action": "ignored"})
             return
 
-        self.task_handler.schedule(
-            notification.kbid, partial(self.process_kb, notification.kbid)
-        )
+        self.task_handler.schedule(notification.kbid, partial(self.process_kb, notification.kbid))
         metrics.total_messages.inc({"type": "shard_creator", "action": "scheduled"})
 
     @metrics.handler_histo.wrap({"type": "shard_creator"})
@@ -105,7 +103,7 @@ class ShardCreatorHandler:
         async with locking.distributed_lock(locking.NEW_SHARD_LOCK.format(kbid=kbid)):
             # remember, a lock will do at least 1+ reads and 1 write.
             # with heavy writes, this adds some simple k/v pressure
-            node, shard_id = choose_node(current_shard)
+            node, shard_id = choose_node(current_shard, use_nidx=True)
             shard: nodereader_pb2.Shard = await node.reader.GetShard(
                 nodereader_pb2.GetShardRequest(shard_id=noderesources_pb2.ShardId(id=shard_id))  # type: ignore
             )

nucliadb/ingest/consumer/utils.py

@@ -48,9 +48,7 @@ class DelayedTaskHandler:
         for task in list(self.outstanding_tasks.values()):
             await task
 
-    def schedule(
-        self, key: str, handler: Callable[[], Coroutine[None, None, None]]
-    ) -> None:
+    def schedule(self, key: str, handler: Callable[[], Coroutine[None, None, None]]) -> None:
         if key in self.to_process:
             # already waiting to process this key, ignore
             return

nucliadb/ingest/fields/base.py

@@ -21,14 +21,20 @@ from __future__ import annotations
 
 import enum
 from datetime import datetime
-from typing import Any, Optional, Type
+from typing import Any, Generic, Optional, Type, TypeVar
 
+from google.protobuf.message import DecodeError, Message
+
+from nucliadb.common import datamanagers
+from nucliadb.ingest.fields.exceptions import InvalidFieldClass, InvalidPBClass
 from nucliadb_protos.resources_pb2 import (
     CloudFile,
     ExtractedTextWrapper,
     ExtractedVectorsWrapper,
+    FieldAuthor,
     FieldComputedMetadata,
     FieldComputedMetadataWrapper,
+    FieldQuestionAnswers,
     FieldQuestionAnswerWrapper,
     LargeComputedMetadata,
     LargeComputedMetadataWrapper,
@@ -36,34 +42,33 @@ from nucliadb_protos.resources_pb2 import (
 )
 from nucliadb_protos.utils_pb2 import ExtractedText, VectorObject
 from nucliadb_protos.writer_pb2 import Error
-
-from nucliadb.ingest.fields.exceptions import InvalidFieldClass, InvalidPBClass
 from nucliadb_utils.storages.storage import Storage, StorageField
 
-KB_RESOURCE_FIELD = "/kbs/{kbid}/r/{uuid}/f/{type}/{field}"
-KB_RESOURCE_ERROR = "/kbs/{kbid}/r/{uuid}/f/{type}/{field}/error"
-
-SUBFIELDFIELDS = ["l", "c"]
+SUBFIELDFIELDS = ("c",)
 
 
 class FieldTypes(str, enum.Enum):
     FIELD_TEXT = "extracted_text"
     FIELD_VECTORS = "extracted_vectors"
+    FIELD_VECTORSET = "{vectorset}/extracted_vectors"
     FIELD_METADATA = "metadata"
     FIELD_LARGE_METADATA = "large_metadata"
     THUMBNAIL = "thumbnail"
     QUESTION_ANSWERS = "question_answers"
 
 
-class Field:
-    pbklass: Optional[Type] = None
+PbType = TypeVar("PbType", bound=Message)
+
+
+class Field(Generic[PbType]):
+    pbklass: Type[PbType]
     type: str = "x"
     value: Optional[Any]
     extracted_text: Optional[ExtractedText]
-    extracted_vectors: Optional[VectorObject]
+    extracted_vectors: dict[Optional[str], VectorObject]
     computed_metadata: Optional[FieldComputedMetadata]
     large_computed_metadata: Optional[LargeComputedMetadata]
-    question_answers: Optional[QuestionAnswers]
+    question_answers: Optional[FieldQuestionAnswers]
 
     def __init__(
         self,
@@ -77,7 +82,7 @@ class Field:
 
         self.value = None
         self.extracted_text: Optional[ExtractedText] = None
-        self.extracted_vectors = None
+        self.extracted_vectors = {}
         self.computed_metadata = None
         self.large_computed_metadata = None
         self.question_answers = None
@@ -112,44 +117,51 @@ class Field:
         return f"{self.uuid}/{self.type}/{self.id}"
 
     def get_storage_field(self, field_type: FieldTypes) -> StorageField:
-        return self.storage.file_extracted(
-            self.kbid, self.uuid, self.type, self.id, field_type.value
-        )
+        return self.storage.file_extracted(self.kbid, self.uuid, self.type, self.id, field_type.value)
+
+    def _get_extracted_vectors_storage_field(self, vectorset: Optional[str] = None) -> StorageField:
+        if vectorset:
+            key = FieldTypes.FIELD_VECTORSET.value.format(vectorset=vectorset)
+        else:
+            key = FieldTypes.FIELD_VECTORS.value
+        return self.storage.file_extracted(self.kbid, self.uuid, self.type, self.id, key)
 
-    async def db_get_value(self):
+    async def db_get_value(self) -> Optional[PbType]:
         if self.value is None:
-            payload = await self.resource.txn.get(
-                KB_RESOURCE_FIELD.format(
-                    kbid=self.kbid, uuid=self.uuid, type=self.type, field=self.id
-                )
+            payload = await datamanagers.fields.get_raw(
+                self.resource.txn,
+                kbid=self.kbid,
+                rid=self.uuid,
+                field_type=self.type,
+                field_id=self.id,
             )
             if payload is None:
-                return
+                return None
 
             self.value = self.pbklass()
             self.value.ParseFromString(payload)
         return self.value
 
     async def db_set_value(self, payload: Any):
-        await self.resource.txn.set(
-            KB_RESOURCE_FIELD.format(
-                kbid=self.kbid, uuid=self.uuid, type=self.type, field=self.id
-            ),
-            payload.SerializeToString(),
+        await datamanagers.fields.set(
+            self.resource.txn,
+            kbid=self.kbid,
+            rid=self.uuid,
+            field_type=self.type,
+            field_id=self.id,
+            value=payload,
         )
         self.value = payload
         self.resource.modified = True
 
     async def delete(self):
-        field_base_key = KB_RESOURCE_FIELD.format(
-            kbid=self.kbid, uuid=self.uuid, type=self.type, field=self.id
+        await datamanagers.fields.delete(
+            self.resource.txn,
+            kbid=self.kbid,
+            rid=self.uuid,
+            field_type=self.type,
+            field_id=self.id,
         )
-        # Make sure we explicitly delete the field and any nested key
-        keys_to_delete = []
-        async for key in self.resource.txn.keys(field_base_key):
-            keys_to_delete.append(key)
-        for key in keys_to_delete:
-            await self.resource.txn.delete(key)
         await self.delete_extracted_text()
         await self.delete_vectors()
         await self.delete_metadata()
@@ -169,9 +181,9 @@ class Field:
         except KeyError:
             pass
 
-    async def delete_vectors(self) -> None:
+    async def delete_vectors(self, vectorset: Optional[str] = None) -> None:
         # Try delete vectors
-        sf = self.get_storage_field(FieldTypes.FIELD_VECTORS)
+        sf = self._get_extracted_vectors_storage_field(vectorset)
         try:
             await self.storage.delete_upload(sf.key, sf.bucket)
         except KeyError:
@@ -185,53 +197,79 @@ class Field:
             pass
 
     async def get_error(self) -> Optional[Error]:
-        payload = await self.resource.txn.get(
-            KB_RESOURCE_ERROR.format(
-                kbid=self.kbid, uuid=self.uuid, type=self.type, field=self.id
-            )
+        return await datamanagers.fields.get_error(
+            self.resource.txn,
+            kbid=self.kbid,
+            rid=self.uuid,
+            field_type=self.type,
+            field_id=self.id,
         )
-        if payload is None:
-            return None
-        pberror = Error()
-        pberror.ParseFromString(payload)
-        return pberror
 
     async def set_error(self, error: Error) -> None:
-        await self.resource.txn.set(
-            KB_RESOURCE_ERROR.format(
-                kbid=self.kbid, uuid=self.uuid, type=self.type, field=self.id
-            ),
-            error.SerializeToString(),
+        await datamanagers.fields.set_error(
+            self.resource.txn,
+            kbid=self.kbid,
+            rid=self.uuid,
+            field_type=self.type,
+            field_id=self.id,
+            error=error,
         )
 
-    async def get_question_answers(self) -> Optional[QuestionAnswers]:
-        if self.question_answers is None:
+    async def get_question_answers(self, force=False) -> Optional[FieldQuestionAnswers]:
+        if self.question_answers is None or force:
             sf = self.get_storage_field(FieldTypes.QUESTION_ANSWERS)
-            payload = await self.storage.download_pb(sf, QuestionAnswers)
+            try:
+                payload = await self.storage.download_pb(sf, FieldQuestionAnswers)
+            except DecodeError:
+                deprecated_payload = await self.storage.download_pb(sf, QuestionAnswers)
+                if deprecated_payload is not None:
+                    payload = FieldQuestionAnswers()
+                    payload.question_answers.CopyFrom(deprecated_payload)
             if payload is not None:
                 self.question_answers = payload
         return self.question_answers
 
     async def set_question_answers(self, payload: FieldQuestionAnswerWrapper) -> None:
+        if self.type in SUBFIELDFIELDS:
+            try:
+                actual_payload: Optional[FieldQuestionAnswers] = await self.get_question_answers(
+                    force=True
+                )
+            except KeyError:
+                actual_payload = None
+        else:
+            actual_payload = None
         sf = self.get_storage_field(FieldTypes.QUESTION_ANSWERS)
 
-        if payload.HasField("file"):
-            raw_payload = await self.storage.downloadbytescf(payload.file)
-            pb = QuestionAnswers()
-            pb.ParseFromString(raw_payload.read())
-            raw_payload.flush()
-            self.question_answers = pb
+        if actual_payload is None:
+            # Its first question answer
+            if payload.HasField("file"):
+                await self.storage.normalize_binary(payload.file, sf)
+            else:
+                await self.storage.upload_pb(sf, payload.question_answers)
+                self.question_answers = payload.question_answers
         else:
-            self.question_answers = payload.question_answers
-
-        await self.storage.upload_pb(sf, self.question_answers)
+            if payload.HasField("file"):
+                raw_payload = await self.storage.downloadbytescf(payload.file)
+                pb = FieldQuestionAnswers()
+                pb.ParseFromString(raw_payload.read())
+                raw_payload.flush()
+                payload.question_answers.CopyFrom(pb)
+            # We know its payload.question_answers
+            for key, value in payload.question_answers.split_question_answers.items():
+                actual_payload.split_question_answers[key] = value
+            for key in payload.question_answers.deleted_splits:
+                if key in actual_payload.split_question_answers:
+                    del actual_payload.split_question_answers[key]
+            if payload.question_answers.HasField("question_answers") != "":
+                actual_payload.question_answers.CopyFrom(payload.question_answers.question_answers)
+            await self.storage.upload_pb(sf, actual_payload)
+            self.question_answers = actual_payload
 
     async def set_extracted_text(self, payload: ExtractedTextWrapper) -> None:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[ExtractedText] = await self.get_extracted_text(
-                    force=True
-                )
+                actual_payload: Optional[ExtractedText] = await self.get_extracted_text(force=True)
             except KeyError:
                 actual_payload = None
         else:
@@ -271,23 +309,21 @@ class Field:
                 self.extracted_text = payload
         return self.extracted_text
 
-    async def set_vectors(
-        self, payload: ExtractedVectorsWrapper
-    ) -> tuple[Optional[VectorObject], bool, list[str]]:
+    async def set_vectors(self, payload: ExtractedVectorsWrapper) -> Optional[VectorObject]:
+        vectorset = payload.vectorset_id or None
         if self.type in SUBFIELDFIELDS:
             try:
                 actual_payload: Optional[VectorObject] = await self.get_vectors(
-                    force=True
+                    vectorset=vectorset,
+                    force=True,
                 )
             except KeyError:
                 actual_payload = None
         else:
             actual_payload = None
 
-        sf = self.get_storage_field(FieldTypes.FIELD_VECTORS)
+        sf = self._get_extracted_vectors_storage_field(vectorset)
         vo: Optional[VectorObject] = None
-        replace_field: bool = True
-        replace_splits = []
         if actual_payload is None:
             # Its first extracted text
             if payload.HasField("file"):
@@ -296,7 +332,7 @@ class Field:
             else:
                 await self.storage.upload_pb(sf, payload.vectors)
                 vo = payload.vectors
-                self.extracted_vectors = payload.vectors
+                self.extracted_vectors[vectorset] = payload.vectors
         else:
             if payload.HasField("file"):
                 raw_payload = await self.storage.downloadbytescf(payload.file)
@@ -304,36 +340,38 @@ class Field:
                 pb.ParseFromString(raw_payload.read())
                 raw_payload.flush()
                 payload.vectors.CopyFrom(pb)
-            vo = payload.vectors
+            vo = actual_payload
             # We know its payload.body
             for key, value in payload.vectors.split_vectors.items():
                 actual_payload.split_vectors[key].CopyFrom(value)
             for key in payload.vectors.deleted_splits:
                 if key in actual_payload.split_vectors:
-                    replace_splits.append(key)
                     del actual_payload.split_vectors[key]
             if len(payload.vectors.vectors.vectors) > 0:
-                replace_field = True
                 actual_payload.vectors.CopyFrom(payload.vectors.vectors)
             await self.storage.upload_pb(sf, actual_payload)
-            self.extracted_vectors = actual_payload
-        return vo, replace_field, replace_splits
-
-    async def get_vectors(self, force=False) -> Optional[VectorObject]:
-        if self.extracted_vectors is None or force:
-            sf = self.get_storage_field(FieldTypes.FIELD_VECTORS)
+            self.extracted_vectors[vectorset] = actual_payload
+        return vo
+
+    async def get_vectors(
+        self, vectorset: Optional[str] = None, force: bool = False
+    ) -> Optional[VectorObject]:
+        # compat with vectorsets coming from protobuffers where no value is
+        # empty string instead of None. This shouldn't be handled here but we
+        # have to make sure it gets the correct vectorset
+        vectorset = vectorset or None
+        if self.extracted_vectors.get(vectorset, None) is None or force:
+            sf = self._get_extracted_vectors_storage_field(vectorset)
             payload = await self.storage.download_pb(sf, VectorObject)
             if payload is not None:
-                self.extracted_vectors = payload
-        return self.extracted_vectors
+                self.extracted_vectors[vectorset] = payload
+        return self.extracted_vectors.get(vectorset, None)
 
-    async def set_field_metadata(
-        self, payload: FieldComputedMetadataWrapper
-    ) -> tuple[FieldComputedMetadata, list[str], dict[str, list[str]]]:
+    async def set_field_metadata(self, payload: FieldComputedMetadataWrapper) -> FieldComputedMetadata:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[FieldComputedMetadata] = (
-                    await self.get_field_metadata(force=True)
+                actual_payload: Optional[FieldComputedMetadata] = await self.get_field_metadata(
+                    force=True
                 )
             except KeyError:
                 actual_payload = None
@@ -359,8 +397,6 @@ class Field:
                 metadata.thumbnail.CopyFrom(cf_split)
             metadata.last_index.FromDatetime(datetime.now())
 
-        replace_field = []
-        replace_splits = {}
         if actual_payload is None:
             # Its first metadata
             await self.storage.upload_pb(sf, payload.metadata)
@@ -371,22 +407,15 @@ class Field:
                 actual_payload.split_metadata[key].CopyFrom(value)
             for key in payload.metadata.deleted_splits:
                 if key in actual_payload.split_metadata:
-                    replace_splits[key] = [
-                        f"{x.start}-{x.end}"
-                        for x in actual_payload.split_metadata[key].paragraphs
-                    ]
                     del actual_payload.split_metadata[key]
             if payload.metadata.metadata:
                 actual_payload.metadata.CopyFrom(payload.metadata.metadata)
-                replace_field = [f"{x.start}-{x.end}" for x in metadata.paragraphs]
             await self.storage.upload_pb(sf, actual_payload)
             self.computed_metadata = actual_payload
 
-        return self.computed_metadata, replace_field, replace_splits
+        return self.computed_metadata
 
-    async def get_field_metadata(
-        self, force: bool = False
-    ) -> Optional[FieldComputedMetadata]:
+    async def get_field_metadata(self, force: bool = False) -> Optional[FieldComputedMetadata]:
         if self.computed_metadata is None or force:
             sf = self.get_storage_field(FieldTypes.FIELD_METADATA)
             payload = await self.storage.download_pb(sf, FieldComputedMetadata)
@@ -397,8 +426,8 @@ class Field:
     async def set_large_field_metadata(self, payload: LargeComputedMetadataWrapper):
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[LargeComputedMetadata] = (
-                    await self.get_large_field_metadata(force=True)
+                actual_payload: Optional[LargeComputedMetadata] = await self.get_large_field_metadata(
+                    force=True
                 )
             except KeyError:
                 actual_payload = None
@@ -434,9 +463,7 @@ class Field:
 
         return self.large_computed_metadata
 
-    async def get_large_field_metadata(
-        self, force: bool = False
-    ) -> Optional[LargeComputedMetadata]:
+    async def get_large_field_metadata(self, force: bool = False) -> Optional[LargeComputedMetadata]:
         if self.large_computed_metadata is None or force:
             sf = self.get_storage_field(FieldTypes.FIELD_LARGE_METADATA)
             payload = await self.storage.download_pb(
@@ -447,6 +474,11 @@ class Field:
                 self.large_computed_metadata = payload
         return self.large_computed_metadata
 
+    async def generated_by(self) -> FieldAuthor:
+        author = FieldAuthor()
+        author.user.SetInParent()
+        return author
+
     def serialize(self):
         return self.value.SerializeToString()
 

nucliadb/ingest/fields/conversation.py

@@ -20,11 +20,9 @@
 import uuid
 from typing import Any, Optional
 
-from nucliadb_protos.resources_pb2 import CloudFile
-from nucliadb_protos.resources_pb2 import Conversation as PBConversation
-from nucliadb_protos.resources_pb2 import FieldConversation
-
 from nucliadb.ingest.fields.base import Field
+from nucliadb_protos.resources_pb2 import CloudFile, FieldConversation
+from nucliadb_protos.resources_pb2 import Conversation as PBConversation
 from nucliadb_utils.storages.storage import StorageField
 
 PAGE_SIZE = 200
@@ -36,7 +34,7 @@ class PageNotFound(Exception):
     pass
 
 
-class Conversation(Field):
+class Conversation(Field[PBConversation]):
     pbklass = PBConversation
     type: str = "c"
     value: dict[int, PBConversation]
@@ -120,6 +118,21 @@ class Conversation(Field):
         except PageNotFound:
             return None
 
+    async def get_full_conversation(self) -> Optional[PBConversation]:
+        """
+        Messages of a conversations may be stored across several pages.
+        This method fetches them all and returns a single complete conversation.
+        """
+        full_conv = PBConversation()
+        n_page = 1
+        while True:
+            page = await self.get_value(page=n_page)
+            if page is None:
+                break
+            full_conv.messages.extend(page.messages)
+            n_page += 1
+        return full_conv
+
     async def get_metadata(self) -> FieldConversation:
         if self.metadata is None:
             payload = await self.resource.txn.get(

nucliadb/ingest/fields/exceptions.py

@@ -28,7 +28,4 @@ class InvalidPBClass(Exception):
     def __init__(self, source: Type, destination: Type):
         self.source = source
         self.destination = destination
-        super().__init__(
-            "Source and destination does not match "
-            f"{self.source} - {self.destination}"
-        )
+        super().__init__("Source and destination does not match " f"{self.source} - {self.destination}")

nucliadb/ingest/fields/file.py

@@ -19,15 +19,14 @@
 #
 from typing import Any, Optional
 
-from nucliadb_protos.resources_pb2 import CloudFile, FieldFile, FileExtractedData
-
 from nucliadb.ingest.fields.base import Field
+from nucliadb_protos.resources_pb2 import CloudFile, FieldFile, FileExtractedData
 from nucliadb_utils.storages.storage import StorageField
 
 FILE_METADATA = "file_metadata"
 
 
-class File(Field):
+class File(Field[FieldFile]):
     pbklass = FieldFile
     value: FieldFile
     type: str = "f"
@@ -52,15 +51,13 @@ class File(Field):
 
         is_external_file = payload.file.source == CloudFile.Source.EXTERNAL
         if not is_external_file:
-            sf: StorageField = self.storage.file_field(
-                self.kbid, self.uuid, self.id, old_cf
-            )
+            sf: StorageField = self.storage.file_field(self.kbid, self.uuid, self.id, old_cf)
             cf: CloudFile = await self.storage.normalize_binary(payload.file, sf)
             payload.file.CopyFrom(cf)
 
         await self.db_set_value(payload)
 
-    async def get_value(self) -> FieldFile:
+    async def get_value(self) -> Optional[FieldFile]:
         return await self.db_get_value()
 
     async def set_file_extracted_data(self, file_extracted_data: FileExtractedData):
@@ -80,17 +77,13 @@ class File(Field):
             cf_file_page_preview: CloudFile = await self.storage.normalize_binary(
                 preview, sf_file_page_preview
             )
-            file_extracted_data.file_pages_previews.pages[page].CopyFrom(
-                cf_file_page_preview
-            )
+            file_extracted_data.file_pages_previews.pages[page].CopyFrom(cf_file_page_preview)
 
         for fileid, origincf in file_extracted_data.file_generated.items():
             sf_generated: StorageField = self.storage.file_extracted(
                 self.kbid, self.uuid, self.type, self.id, f"generated/{fileid}"
             )
-            cf_generated: CloudFile = await self.storage.normalize_binary(
-                origincf, sf_generated
-            )
+            cf_generated: CloudFile = await self.storage.normalize_binary(origincf, sf_generated)
             file_extracted_data.file_generated[fileid].CopyFrom(cf_generated)
 
         if file_extracted_data.HasField("file_thumbnail"):
@@ -113,7 +106,5 @@ class File(Field):
             sf: StorageField = self.storage.file_extracted(
                 self.kbid, self.uuid, self.type, self.id, FILE_METADATA
             )
-            self.file_extracted_data = await self.storage.download_pb(
-                sf, FileExtractedData
-            )
+            self.file_extracted_data = await self.storage.download_pb(sf, FileExtractedData)
         return self.file_extracted_data