nucliadb 6.9.1.post5192__py3-none-any.whl → 6.10.0.post5705__py3-none-any.whl

nucliadb/ingest/fields/base.py

@@ -24,7 +24,7 @@ import enum
 import logging
 from collections import defaultdict
 from datetime import datetime
-from typing import TYPE_CHECKING, Any, Generic, Optional, Type, TypeVar
+from typing import TYPE_CHECKING, Any, Generic, TypeVar
 
 from google.protobuf.message import DecodeError, Message
 
@@ -47,10 +47,8 @@ from nucliadb_protos.resources_pb2 import (
 )
 from nucliadb_protos.utils_pb2 import ExtractedText, VectorObject
 from nucliadb_protos.writer_pb2 import Error, FieldStatus
-from nucliadb_utils import const
 from nucliadb_utils.storages.exceptions import CouldNotCopyNotFound
 from nucliadb_utils.storages.storage import Storage, StorageField
-from nucliadb_utils.utilities import has_feature
 
 logger = logging.getLogger(__name__)
 
@@ -77,27 +75,27 @@ PbType = TypeVar("PbType", bound=Message)
 
 
 class Field(Generic[PbType]):
-    pbklass: Type[PbType]
+    pbklass: type[PbType]
     type: str = "x"
-    value: Optional[Any]
-    extracted_text: Optional[ExtractedText]
-    extracted_vectors: dict[Optional[str], VectorObject]
-    computed_metadata: Optional[FieldComputedMetadata]
-    large_computed_metadata: Optional[LargeComputedMetadata]
-    question_answers: Optional[FieldQuestionAnswers]
+    value: Any | None
+    extracted_text: ExtractedText | None
+    extracted_vectors: dict[str | None, VectorObject]
+    computed_metadata: FieldComputedMetadata | None
+    large_computed_metadata: LargeComputedMetadata | None
+    question_answers: FieldQuestionAnswers | None
 
     def __init__(
         self,
         id: str,
         resource: Resource,
-        pb: Optional[Any] = None,
-        value: Optional[Any] = None,
+        pb: Any | None = None,
+        value: Any | None = None,
     ):
         if self.pbklass is None:
             raise InvalidFieldClass()
 
         self.value = None
-        self.extracted_text: Optional[ExtractedText] = None
+        self.extracted_text: ExtractedText | None = None
         self.extracted_vectors = {}
         self.computed_metadata = None
         self.large_computed_metadata = None
@@ -120,7 +118,7 @@ class Field(Generic[PbType]):
 
     @property
     def kbid(self) -> str:
-        return self.resource.kb.kbid
+        return self.resource.kbid
 
     @property
     def uuid(self) -> str:
@@ -161,7 +159,7 @@ class Field(Generic[PbType]):
 
         return self.storage.file_extracted(self.kbid, self.uuid, self.type, self.id, key)
 
-    async def db_get_value(self) -> Optional[PbType]:
+    async def db_get_value(self) -> PbType | None:
         if self.value is None:
             payload = await datamanagers.fields.get_raw(
                 self.resource.txn,
@@ -224,21 +222,6 @@ class Field(Generic[PbType]):
     ) -> None:
         # Try delete vectors
         sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
-
-        if has_feature(const.Features.DEBUG_MISSING_VECTORS):
-            # This is a very chatty log. It is just a temporary hint while debugging an issue.
-            logger.info(
-                "Deleting vectors from storage",
-                extra={
-                    "kbid": self.kbid,
-                    "rid": self.resource.uuid,
-                    "field": f"{self.type}/{self.id}",
-                    "vectorset": vectorset,
-                    "storage_key_kind": storage_key_kind,
-                    "key": sf.key,
-                    "bucket": sf.bucket,
-                },
-            )
         try:
             await self.storage.delete_upload(sf.key, sf.bucket)
         except KeyError:
@@ -251,7 +234,7 @@ class Field(Generic[PbType]):
         except KeyError:
             pass
 
-    async def get_error(self) -> Optional[Error]:
+    async def get_error(self) -> Error | None:
         return await datamanagers.fields.get_error(
             self.resource.txn,
             kbid=self.kbid,
@@ -270,7 +253,7 @@ class Field(Generic[PbType]):
             error=error,
         )
 
-    async def get_status(self) -> Optional[FieldStatus]:
+    async def get_status(self) -> FieldStatus | None:
         return await datamanagers.fields.get_status(
             self.resource.txn,
             kbid=self.kbid,
@@ -289,7 +272,7 @@ class Field(Generic[PbType]):
             status=status,
         )
 
-    async def get_question_answers(self, force=False) -> Optional[FieldQuestionAnswers]:
+    async def get_question_answers(self, force=False) -> FieldQuestionAnswers | None:
         if self.question_answers is None or force:
             sf = self.get_storage_field(FieldTypes.QUESTION_ANSWERS)
             try:
@@ -306,9 +289,7 @@ class Field(Generic[PbType]):
     async def set_question_answers(self, payload: FieldQuestionAnswerWrapper) -> None:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[FieldQuestionAnswers] = await self.get_question_answers(
-                    force=True
-                )
+                actual_payload: FieldQuestionAnswers | None = await self.get_question_answers(force=True)
             except KeyError:
                 actual_payload = None
         else:
@@ -341,7 +322,7 @@ class Field(Generic[PbType]):
             self.question_answers = actual_payload
 
     async def set_extracted_text(self, payload: ExtractedTextWrapper) -> None:
-        actual_payload: Optional[ExtractedText] = None
+        actual_payload: ExtractedText | None = None
         if self.type in SUBFIELDFIELDS:
             # Try to get the previously extracted text protobuf if it exists so we can merge it with the new splits
             # coming from the processing payload.
@@ -392,7 +373,7 @@ class Field(Generic[PbType]):
             await self.storage.upload_pb(sf, actual_payload)
             self.extracted_text = actual_payload
 
-    async def get_extracted_text(self, force=False) -> Optional[ExtractedText]:
+    async def get_extracted_text(self, force=False) -> ExtractedText | None:
         if self.extracted_text is None or force:
             async with self.locks["extracted_text"]:
                 # Value could have been fetched while waiting for the lock
@@ -408,10 +389,10 @@ class Field(Generic[PbType]):
         payload: ExtractedVectorsWrapper,
         vectorset: str,
         storage_key_kind: VectorSetConfig.StorageKeyKind.ValueType,
-    ) -> Optional[VectorObject]:
+    ) -> VectorObject | None:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[VectorObject] = await self.get_vectors(
+                actual_payload: VectorObject | None = await self.get_vectors(
                     vectorset=vectorset,
                     storage_key_kind=storage_key_kind,
                     force=True,
@@ -422,7 +403,7 @@ class Field(Generic[PbType]):
             actual_payload = None
 
         sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
-        vo: Optional[VectorObject] = None
+        vo: VectorObject | None = None
         if actual_payload is None:
             # Its first extracted vectors
             if payload.HasField("file"):
@@ -474,7 +455,7 @@ class Field(Generic[PbType]):
         vectorset: str,
         storage_key_kind: VectorSetConfig.StorageKeyKind.ValueType,
         force: bool = False,
-    ) -> Optional[VectorObject]:
+    ) -> VectorObject | None:
         if self.extracted_vectors.get(vectorset, None) is None or force:
             sf = self._get_extracted_vectors_storage_field(vectorset, storage_key_kind)
             payload = await self.storage.download_pb(sf, VectorObject)
@@ -485,9 +466,7 @@ class Field(Generic[PbType]):
     async def set_field_metadata(self, payload: FieldComputedMetadataWrapper) -> FieldComputedMetadata:
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[FieldComputedMetadata] = await self.get_field_metadata(
-                    force=True
-                )
+                actual_payload: FieldComputedMetadata | None = await self.get_field_metadata(force=True)
             except KeyError:
                 actual_payload = None
         else:
@@ -530,7 +509,7 @@ class Field(Generic[PbType]):
 
         return self.computed_metadata
 
-    async def get_field_metadata(self, force: bool = False) -> Optional[FieldComputedMetadata]:
+    async def get_field_metadata(self, force: bool = False) -> FieldComputedMetadata | None:
         if self.computed_metadata is None or force:
             async with self.locks["field_metadata"]:
                 # Value could have been fetched while waiting for the lock
@@ -544,7 +523,7 @@ class Field(Generic[PbType]):
     async def set_large_field_metadata(self, payload: LargeComputedMetadataWrapper):
         if self.type in SUBFIELDFIELDS:
             try:
-                actual_payload: Optional[LargeComputedMetadata] = await self.get_large_field_metadata(
+                actual_payload: LargeComputedMetadata | None = await self.get_large_field_metadata(
                     force=True
                 )
             except KeyError:
@@ -554,7 +533,7 @@ class Field(Generic[PbType]):
 
         sf = self.get_storage_field(FieldTypes.FIELD_LARGE_METADATA)
 
-        new_payload: Optional[LargeComputedMetadata] = None
+        new_payload: LargeComputedMetadata | None = None
         if payload.HasField("file"):
             new_payload = LargeComputedMetadata()
             data = await self.storage.downloadbytescf(payload.file)
@@ -581,7 +560,7 @@ class Field(Generic[PbType]):
 
         return self.large_computed_metadata
 
-    async def get_large_field_metadata(self, force: bool = False) -> Optional[LargeComputedMetadata]:
+    async def get_large_field_metadata(self, force: bool = False) -> LargeComputedMetadata | None:
         if self.large_computed_metadata is None or force:
             sf = self.get_storage_field(FieldTypes.FIELD_LARGE_METADATA)
             payload = await self.storage.download_pb(

nucliadb/ingest/fields/conversation.py

@@ -18,14 +18,14 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 import uuid
-from typing import Any, Optional
+from typing import Any
 
 from nucliadb.ingest.fields.base import Field
-from nucliadb_protos.resources_pb2 import CloudFile, FieldConversation, SplitMetadata, SplitsMetadata
+from nucliadb_protos.resources_pb2 import CloudFile, FieldConversation, SplitsMetadata
 from nucliadb_protos.resources_pb2 import Conversation as PBConversation
 from nucliadb_utils.storages.storage import StorageField
 
-MAX_CONVERSATION_MESSAGES = 50 * 1024
+MAX_CONVERSATION_MESSAGES = None  # No limit
 
 PAGE_SIZE = 200
 
@@ -42,7 +42,7 @@ class Conversation(Field[PBConversation]):
     pbklass = PBConversation
     type: str = "c"
     value: dict[int, PBConversation]
-    metadata: Optional[FieldConversation]
+    metadata: FieldConversation | None
 
     _created: bool = False
 
@@ -50,12 +50,12 @@ class Conversation(Field[PBConversation]):
         self,
         id: str,
         resource: Any,
-        pb: Optional[Any] = None,
-        value: Optional[dict[int, PBConversation]] = None,
+        pb: Any | None = None,
+        value: dict[int, PBConversation] | None = None,
     ):
-        super(Conversation, self).__init__(id, resource, pb, value)
+        super().__init__(id, resource, pb, value)
         self.value = {}
-        self._splits_metadata: Optional[SplitsMetadata] = None
+        self._splits_metadata: SplitsMetadata | None = None
         self.metadata = None
 
     async def delete_value(self):
@@ -76,7 +76,7 @@ class Conversation(Field[PBConversation]):
         metadata.split_strategy = payload.split_strategy
 
         # Get the last page if it exists
-        last_page: Optional[PBConversation] = None
+        last_page: PBConversation | None = None
         if self._created is False and metadata.pages > 0:
             try:
                 last_page = await self.db_get_value(page=metadata.pages)
@@ -92,7 +92,7 @@ class Conversation(Field[PBConversation]):
         # to support the hybrid-onprem deployment as the attachments must be stored
         # at the storage services of the client's premises.
         for message in payload.messages:
-            self._splits_metadata.metadata.setdefault(message.ident, SplitMetadata())
+            self._splits_metadata.metadata.get_or_create(message.ident)
             new_message_files = []
             for idx, file in enumerate(message.content.attachments):
                 if self.storage.needs_move(file, self.kbid):
@@ -138,7 +138,7 @@ class Conversation(Field[PBConversation]):
         await self.db_set_metadata(metadata)
         await self.set_splits_metadata(self._splits_metadata)
 
-    async def get_value(self, page: Optional[int] = None) -> Optional[PBConversation]:
+    async def get_value(self, page: int | None = None) -> PBConversation | None:
         # If no page was requested, force fetch of metadata
         # and set the page to the last page
         if page is None and self.metadata is None:
@@ -153,7 +153,7 @@ class Conversation(Field[PBConversation]):
         except PageNotFound:
             return None
 
-    async def get_full_conversation(self) -> Optional[PBConversation]:
+    async def get_full_conversation(self) -> PBConversation | None:
         """
         Messages of a conversations may be stored across several pages.
         This method fetches them all and returns a single complete conversation.

nucliadb/ingest/fields/exceptions.py

@@ -17,7 +17,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Type
 
 
 class InvalidFieldClass(Exception):
@@ -25,7 +24,7 @@ class InvalidFieldClass(Exception):
 
 
 class InvalidPBClass(Exception):
-    def __init__(self, source: Type, destination: Type):
+    def __init__(self, source: type, destination: type):
         self.source = source
         self.destination = destination
         super().__init__(f"Source and destination does not match {self.source} - {self.destination}")

nucliadb/ingest/fields/file.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Any, Optional
+from typing import Any
 
 from nucliadb.ingest.fields.base import Field
 from nucliadb_protos.resources_pb2 import CloudFile, FieldFile, FileExtractedData
@@ -30,22 +30,22 @@ class File(Field[FieldFile]):
     pbklass = FieldFile
     value: FieldFile
     type: str = "f"
-    file_extracted_data: Optional[FileExtractedData]
+    file_extracted_data: FileExtractedData | None
 
     def __init__(
         self,
         id: str,
         resource: Any,
-        pb: Optional[Any] = None,
-        value: Optional[str] = None,
+        pb: Any | None = None,
+        value: str | None = None,
     ):
-        super(File, self).__init__(id, resource, pb, value)
+        super().__init__(id, resource, pb, value)
         self.file_extracted_data = None
 
     async def set_value(self, payload: FieldFile):
         old_file = await self.get_value()
         if old_file is None:
-            old_cf: Optional[CloudFile] = None
+            old_cf: CloudFile | None = None
         else:
             old_cf = old_file.file
 
@@ -57,7 +57,7 @@ class File(Field[FieldFile]):
 
         await self.db_set_value(payload)
 
-    async def get_value(self) -> Optional[FieldFile]:
+    async def get_value(self) -> FieldFile | None:
         return await self.db_get_value()
 
     async def set_file_extracted_data(self, file_extracted_data: FileExtractedData):
@@ -101,10 +101,24 @@ class File(Field[FieldFile]):
         await self.storage.upload_pb(sf, file_extracted_data)
         self.file_extracted_data = file_extracted_data
 
-    async def get_file_extracted_data(self) -> Optional[FileExtractedData]:
+    async def get_file_extracted_data(self) -> FileExtractedData | None:
         if self.file_extracted_data is None:
             sf: StorageField = self.storage.file_extracted(
                 self.kbid, self.uuid, self.type, self.id, FILE_METADATA
             )
             self.file_extracted_data = await self.storage.download_pb(sf, FileExtractedData)
         return self.file_extracted_data
+
+    async def thumbnail(self) -> StorageField | None:
+        """Access the file field thumbnail."""
+        fed = await self.get_file_extracted_data()
+        if fed is None:
+            return None
+        if not fed.HasField("file_thumbnail"):
+            return None
+
+        sf: StorageField = self.storage.file_extracted(
+            self.kbid, self.uuid, self.type, self.id, "file_thumbnail"
+        )
+        sf.field = fed.file_thumbnail
+        return sf

nucliadb/ingest/fields/link.py

@@ -17,7 +17,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from typing import Any, Optional
+from typing import Any
 
 from nucliadb.ingest.fields.base import Field
 from nucliadb_protos.resources_pb2 import CloudFile, FieldLink, LinkExtractedData
@@ -30,22 +30,22 @@ class Link(Field[FieldLink]):
     pbklass = FieldLink
     value: FieldLink
     type: str = "u"
-    link_extracted_data: Optional[LinkExtractedData]
+    link_extracted_data: LinkExtractedData | None
 
     def __init__(
         self,
         id: str,
         resource: Any,
-        pb: Optional[Any] = None,
-        value: Optional[str] = None,
+        pb: Any | None = None,
+        value: str | None = None,
     ):
-        super(Link, self).__init__(id, resource, pb, value)
+        super().__init__(id, resource, pb, value)
         self.link_extracted_data = None
 
     async def set_value(self, payload: FieldLink):
         await self.db_set_value(payload)
 
-    async def get_value(self) -> Optional[FieldLink]:
+    async def get_value(self) -> FieldLink | None:
         return await self.db_get_value()
 
     async def set_link_extracted_data(self, link_extracted_data: LinkExtractedData):
@@ -88,7 +88,7 @@ class Link(Field[FieldLink]):
         await self.storage.upload_pb(sf, link_extracted_data)
         self.link_extracted_data = link_extracted_data
 
-    async def get_link_extracted_data(self) -> Optional[LinkExtractedData]:
+    async def get_link_extracted_data(self) -> LinkExtractedData | None:
         if self.link_extracted_data is None:
             sf: StorageField = self.storage.file_extracted(
                 self.kbid, self.uuid, self.type, self.id, LINK_METADATA

nucliadb/ingest/fields/text.py

@@ -19,7 +19,6 @@
 #
 
 import hashlib
-from typing import Optional
 
 from nucliadb.ingest.fields.base import Field
 from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
@@ -39,8 +38,8 @@ class Text(Field[FieldText]):
 
     async def set_value(self, payload: FieldText):
         if payload.md5 == "":
-            payload.md5 = hashlib.md5(payload.body.encode()).hexdigest()
+            payload.md5 = hashlib.md5(payload.body.encode(), usedforsecurity=False).hexdigest()
         await self.db_set_value(payload)
 
-    async def get_value(self) -> Optional[FieldText]:
+    async def get_value(self) -> FieldText | None:
         return await self.db_get_value()