nucliadb 2.46.1.post382__py3-none-any.whl → 6.2.1.post2777__py3-none-any.whl

nucliadb/ingest/serialize.py

@@ -23,28 +23,23 @@ from typing import Optional
 import nucliadb_models as models
 from nucliadb.common.maindb.driver import Transaction
 from nucliadb.common.maindb.utils import get_driver
+from nucliadb.common.models_utils import from_proto
 from nucliadb.ingest.fields.base import Field
 from nucliadb.ingest.fields.conversation import Conversation
 from nucliadb.ingest.fields.file import File
 from nucliadb.ingest.fields.link import Link
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
 from nucliadb.ingest.orm.resource import Resource as ORMResource
-from nucliadb_models.common import FIELD_TYPES_MAP, FieldTypeName
+from nucliadb_models.common import FieldTypeName
 from nucliadb_models.resource import (
     ConversationFieldData,
     ConversationFieldExtractedData,
-    DatetimeFieldData,
-    DatetimeFieldExtractedData,
     Error,
     ExtractedDataType,
     ExtractedDataTypeName,
     FileFieldData,
     FileFieldExtractedData,
     GenericFieldData,
-    KeywordsetFieldData,
-    KeywordsetFieldExtractedData,
-    LayoutFieldData,
-    LayoutFieldExtractedData,
     LinkFieldData,
     LinkFieldExtractedData,
     QueueType,
@@ -55,7 +50,6 @@ from nucliadb_models.resource import (
 )
 from nucliadb_models.search import ResourceProperties
 from nucliadb_models.security import ResourceSecurity
-from nucliadb_models.vectors import UserVectorSet
 from nucliadb_utils.utilities import get_storage
 
 
@@ -71,41 +65,32 @@ async def set_resource_field_extracted_data(
     if ExtractedDataTypeName.TEXT in wanted_extracted_data:
         data_et = await field.get_extracted_text()
         if data_et is not None:
-            field_data.text = models.ExtractedText.from_message(data_et)
+            field_data.text = from_proto.extracted_text(data_et)
 
     metadata_wanted = ExtractedDataTypeName.METADATA in wanted_extracted_data
-    shortened_metadata_wanted = (
-        ExtractedDataTypeName.SHORTENED_METADATA in wanted_extracted_data
-    )
+    shortened_metadata_wanted = ExtractedDataTypeName.SHORTENED_METADATA in wanted_extracted_data
     if metadata_wanted or shortened_metadata_wanted:
         data_fcm = await field.get_field_metadata()
 
         if data_fcm is not None:
-            field_data.metadata = models.FieldComputedMetadata.from_message(
+            field_data.metadata = from_proto.field_computed_metadata(
                 data_fcm, shortened=shortened_metadata_wanted and not metadata_wanted
             )
 
     if ExtractedDataTypeName.LARGE_METADATA in wanted_extracted_data:
         data_lcm = await field.get_large_field_metadata()
         if data_lcm is not None:
-            field_data.large_metadata = models.LargeComputedMetadata.from_message(
-                data_lcm
-            )
+            field_data.large_metadata = from_proto.large_computed_metadata(data_lcm)
 
     if ExtractedDataTypeName.VECTOR in wanted_extracted_data:
         data_vec = await field.get_vectors()
         if data_vec is not None:
-            field_data.vectors = models.VectorObject.from_message(data_vec)
-
-    if ExtractedDataTypeName.USERVECTORS in wanted_extracted_data:
-        user_data_vec = await field.get_user_vectors()
-        if user_data_vec is not None:
-            field_data.uservectors = UserVectorSet.from_message(user_data_vec)
+            field_data.vectors = from_proto.vector_object(data_vec)
 
     if ExtractedDataTypeName.QA in wanted_extracted_data:
         qa = await field.get_question_answers()
         if qa is not None:
-            field_data.question_answers = models.QuestionAnswers.from_message(qa)
+            field_data.question_answers = from_proto.field_question_answers(qa)
 
     if (
         isinstance(field, File)
@@ -114,7 +99,7 @@ async def set_resource_field_extracted_data(
     ):
         data_fed = await field.get_file_extracted_data()
         if data_fed is not None:
-            field_data.file = models.FileExtractedData.from_message(data_fed)
+            field_data.file = from_proto.file_extracted_data(data_fed)
 
     if (
         isinstance(field, Link)
@@ -123,7 +108,7 @@ async def set_resource_field_extracted_data(
     ):
         data_led = await field.get_link_extracted_data()
         if data_led is not None:
-            field_data.link = models.LinkExtractedData.from_message(data_led)
+            field_data.link = from_proto.link_extracted_data(data_led)
 
 
 async def serialize(
@@ -136,7 +121,7 @@ async def serialize(
     slug: Optional[str] = None,
 ) -> Optional[Resource]:
     driver = get_driver()
-    async with driver.transaction(wait_for_abort=False, read_only=True) as txn:
+    async with driver.transaction(read_only=True) as txn:
         return await managed_serialize(
             txn,
             kbid,
@@ -159,9 +144,7 @@ async def managed_serialize(
     service_name: Optional[str] = None,
     slug: Optional[str] = None,
 ) -> Optional[Resource]:
-    orm_resource = await get_orm_resource(
-        txn, kbid, rid=rid, slug=slug, service_name=service_name
-    )
+    orm_resource = await get_orm_resource(txn, kbid, rid=rid, slug=slug, service_name=service_name)
     if orm_resource is None:
         return None
 
@@ -169,9 +152,7 @@ async def managed_serialize(
 
     include_values = ResourceProperties.VALUES in show
 
-    include_extracted_data = (
-        ResourceProperties.EXTRACTED in show and extracted is not []
-    )
+    include_extracted_data = ResourceProperties.EXTRACTED in show and extracted is not []
 
     if ResourceProperties.BASIC in show:
         await orm_resource.get_basic()
@@ -181,8 +162,8 @@ async def managed_serialize(
             resource.title = orm_resource.basic.title
             resource.summary = orm_resource.basic.summary
             resource.icon = orm_resource.basic.icon
-            resource.layout = orm_resource.basic.layout
             resource.thumbnail = orm_resource.basic.thumbnail
+            resource.hidden = orm_resource.basic.hidden
             resource.created = (
                 orm_resource.basic.created.ToDatetime()
                 if orm_resource.basic.HasField("created")
@@ -194,49 +175,37 @@ async def managed_serialize(
                 else None
             )
 
-            resource.metadata = models.Metadata.from_message(
-                orm_resource.basic.metadata
-            )
-            resource.usermetadata = models.UserMetadata.from_message(
-                orm_resource.basic.usermetadata
-            )
+            resource.metadata = from_proto.metadata(orm_resource.basic.metadata)
+            resource.usermetadata = from_proto.user_metadata(orm_resource.basic.usermetadata)
             resource.fieldmetadata = [
-                models.UserFieldMetadata.from_message(fm)
-                for fm in orm_resource.basic.fieldmetadata
+                from_proto.user_field_metadata(fm) for fm in orm_resource.basic.fieldmetadata
             ]
-            resource.computedmetadata = models.ComputedMetadata.from_message(
-                orm_resource.basic.computedmetadata
-            )
+            resource.computedmetadata = from_proto.computed_metadata(orm_resource.basic.computedmetadata)
 
             resource.last_seqid = orm_resource.basic.last_seqid
 
             # 0 on the proto means it was not ever set, as first valid value for this field will allways be 1
             resource.last_account_seq = (
-                orm_resource.basic.last_account_seq
-                if orm_resource.basic.last_account_seq != 0
-                else None
+                orm_resource.basic.last_account_seq if orm_resource.basic.last_account_seq != 0 else None
             )
-            resource.queue = QueueType[
-                orm_resource.basic.QueueType.Name(orm_resource.basic.queue)
-            ]
+            resource.queue = QueueType[orm_resource.basic.QueueType.Name(orm_resource.basic.queue)]
 
     if ResourceProperties.RELATIONS in show:
         await orm_resource.get_relations()
         if orm_resource.relations is not None:
             resource.relations = [
-                models.Relation.from_message(relation)
-                for relation in orm_resource.relations.relations
+                from_proto.relation(relation) for relation in orm_resource.relations.relations
             ]
 
     if ResourceProperties.ORIGIN in show:
         await orm_resource.get_origin()
         if orm_resource.origin is not None:
-            resource.origin = models.Origin.from_message(orm_resource.origin)
+            resource.origin = from_proto.origin(orm_resource.origin)
 
     if ResourceProperties.EXTRA in show:
         await orm_resource.get_extra()
         if orm_resource.extra is not None:
-            resource.extra = models.Extra.from_message(orm_resource.extra)
+            resource.extra = from_proto.extra(orm_resource.extra)
 
     include_errors = ResourceProperties.ERRORS in show
 
@@ -247,11 +216,11 @@ async def managed_serialize(
             for gid in orm_resource.security.access_groups:
                 resource.security.access_groups.append(gid)
 
-    if field_type_filter and (include_values or include_extracted_data):
+    if (field_type_filter and (include_values or include_extracted_data)) or include_errors:
         await orm_resource.get_fields()
         resource.data = ResourceData()
-        for (field_type, field_id), field in orm_resource.fields.items():
-            field_type_name = FIELD_TYPES_MAP[field_type]
+        for (field_type, _), field in orm_resource.fields.items():
+            field_type_name = from_proto.field_type_name(field_type)
             if field_type_name not in field_type_filter:
                 continue
 
@@ -266,18 +235,12 @@ async def managed_serialize(
                 if field.id not in resource.data.texts:
                     resource.data.texts[field.id] = TextFieldData()
                 if include_value:
-                    serialized_value = (
-                        models.FieldText.from_message(value)
-                        if value is not None
-                        else None
-                    )
+                    serialized_value = from_proto.field_text(value) if value is not None else None
                     resource.data.texts[field.id].value = serialized_value
                 if include_errors:
                     error = await field.get_error()
                     if error is not None:
-                        resource.data.texts[field.id].error = Error(
-                            body=error.error, code=error.code
-                        )
+                        resource.data.texts[field.id].error = Error(body=error.error, code=error.code)
                 if include_extracted_data:
                     resource.data.texts[field.id].extracted = TextFieldExtractedData()
                     await set_resource_field_extracted_data(
@@ -293,20 +256,14 @@ async def managed_serialize(
                     resource.data.files[field.id] = FileFieldData()
                 if include_value:
                     if value is not None:
-                        resource.data.files[
-                            field.id
-                        ].value = models.FieldFile.from_message(
-                            value  # type: ignore
-                        )
+                        resource.data.files[field.id].value = from_proto.field_file(value)
                     else:
                         resource.data.files[field.id].value = None
 
                 if include_errors:
                     error = await field.get_error()
                     if error is not None:
-                        resource.data.files[field.id].error = Error(
-                            body=error.error, code=error.code
-                        )
+                        resource.data.files[field.id].error = Error(body=error.error, code=error.code)
 
                 if include_extracted_data:
                     resource.data.files[field.id].extracted = FileFieldExtractedData()
@@ -322,16 +279,12 @@ async def managed_serialize(
                 if field.id not in resource.data.links:
                     resource.data.links[field.id] = LinkFieldData()
                 if include_value and value is not None:
-                    resource.data.links[field.id].value = models.FieldLink.from_message(
-                        value
-                    )
+                    resource.data.links[field.id].value = from_proto.field_link(value)
 
                 if include_errors:
                     error = await field.get_error()
                     if error is not None:
-                        resource.data.links[field.id].error = Error(
-                            body=error.error, code=error.code
-                        )
+                        resource.data.links[field.id].error = Error(body=error.error, code=error.code)
 
                 if include_extracted_data:
                     resource.data.links[field.id].extracted = LinkFieldExtractedData()
@@ -341,33 +294,6 @@ async def managed_serialize(
                         field_type_name,
                         extracted,
                     )
-            elif field_type_name is FieldTypeName.LAYOUT:
-                if resource.data.layouts is None:
-                    resource.data.layouts = {}
-                if field.id not in resource.data.layouts:
-                    resource.data.layouts[field.id] = LayoutFieldData()
-                if include_value:
-                    resource.data.layouts[
-                        field.id
-                    ].value = models.FieldLayout.from_message(
-                        value  # type: ignore
-                    )
-                if include_errors:
-                    error = await field.get_error()
-                    if error is not None:
-                        resource.data.layouts[field.id].error = Error(
-                            body=error.error, code=error.code
-                        )
-                if include_extracted_data:
-                    resource.data.layouts[
-                        field.id
-                    ].extracted = LayoutFieldExtractedData()
-                    await set_resource_field_extracted_data(
-                        field,
-                        resource.data.layouts[field.id].extracted,
-                        field_type_name,
-                        extracted,
-                    )
             elif field_type_name is FieldTypeName.CONVERSATION:
                 if resource.data.conversations is None:
                     resource.data.conversations = {}
@@ -381,73 +307,15 @@ async def managed_serialize(
                         )
                 if include_value and isinstance(field, Conversation):
                     value = await field.get_metadata()
-                    resource.data.conversations[
-                        field.id
-                    ].value = models.FieldConversation.from_message(value)
+                    resource.data.conversations[field.id].value = from_proto.field_conversation(value)
                 if include_extracted_data:
-                    resource.data.conversations[
-                        field.id
-                    ].extracted = ConversationFieldExtractedData()
+                    resource.data.conversations[field.id].extracted = ConversationFieldExtractedData()
                     await set_resource_field_extracted_data(
                         field,
                         resource.data.conversations[field.id].extracted,
                         field_type_name,
                         extracted,
                     )
-            elif field_type_name is FieldTypeName.DATETIME:
-                if resource.data.datetimes is None:
-                    resource.data.datetimes = {}
-                if field.id not in resource.data.datetimes:
-                    resource.data.datetimes[field.id] = DatetimeFieldData()
-                if include_errors:
-                    error = await field.get_error()
-                    if error is not None:
-                        resource.data.datetimes[field.id].error = Error(
-                            body=error.error, code=error.code
-                        )
-                if include_value:
-                    resource.data.datetimes[
-                        field.id
-                    ].value = models.FieldDatetime.from_message(
-                        value  # type: ignore
-                    )
-                if include_extracted_data:
-                    resource.data.datetimes[
-                        field.id
-                    ].extracted = DatetimeFieldExtractedData()
-                    await set_resource_field_extracted_data(
-                        field,
-                        resource.data.datetimes[field.id].extracted,
-                        field_type_name,
-                        extracted,
-                    )
-            elif field_type_name is FieldTypeName.KEYWORDSET:
-                if resource.data.keywordsets is None:
-                    resource.data.keywordsets = {field.id: KeywordsetFieldData()}
-                if field.id not in resource.data.keywordsets:
-                    resource.data.keywordsets[field.id] = KeywordsetFieldData()
-                if include_errors:
-                    error = await field.get_error()
-                    if error is not None:
-                        resource.data.keywordsets[field.id].error = Error(
-                            body=error.error, code=error.code
-                        )
-                if include_value:
-                    resource.data.keywordsets[
-                        field.id
-                    ].value = models.FieldKeywordset.from_message(
-                        value  # type: ignore
-                    )
-                if include_extracted_data:
-                    resource.data.keywordsets[
-                        field.id
-                    ].extracted = KeywordsetFieldExtractedData()
-                    await set_resource_field_extracted_data(
-                        field,
-                        resource.data.keywordsets[field.id].extracted,
-                        field_type_name,
-                        extracted,
-                    )
             elif field_type_name is FieldTypeName.GENERIC:
                 if resource.data.generics is None:
                     resource.data.generics = {}
@@ -458,14 +326,10 @@ async def managed_serialize(
                 if include_errors:
                     error = await field.get_error()
                     if error is not None:
-                        resource.data.generics[field.id].error = Error(
-                            body=error.error, code=error.code
-                        )
+                        resource.data.generics[field.id].error = Error(body=error.error, code=error.code)
                 if include_extracted_data:
                     resource.data.generics[field.id].extracted = TextFieldExtractedData(
-                        text=models.ExtractedText(
-                            text=resource.data.generics[field.id].value
-                        )
+                        text=models.ExtractedText(text=resource.data.generics[field.id].value)
                     )
     return resource
 
@@ -502,6 +366,6 @@ async def get_resource_uuid_by_slug(
 ) -> Optional[str]:
     storage = await get_storage(service_name=service_name)
     driver = get_driver()
-    async with driver.transaction() as txn:
+    async with driver.transaction(read_only=True) as txn:
         kb = KnowledgeBox(txn, storage, kbid)
         return await kb.get_resource_uuid_by_slug(slug)

nucliadb/ingest/service/__init__.py

@@ -48,9 +48,7 @@ async def start_grpc(service_name: Optional[str] = None):
 
     await server.start()
 
-    logger.info(
-        f"======= Ingest GRPC running on http://0.0.0.0:{settings.grpc_port}/ ======"
-    )
+    logger.info(f"======= Ingest GRPC running on http://0.0.0.0:{settings.grpc_port}/ ======")
 
     async def finalizer():
         await health_check_finalizer()