nucliadb 4.0.0.post542__py3-none-any.whl → 6.2.1.post2798__py3-none-any.whl

nucliadb/writer/back_pressure.py

@@ -28,7 +28,6 @@ from typing import Optional
 from async_lru import alru_cache
 from cachetools import TTLCache
 from fastapi import HTTPException, Request
-from nucliadb_protos.writer_pb2 import ShardObject
 
 from nucliadb.common import datamanagers
 from nucliadb.common.cluster.manager import get_index_nodes
@@ -37,6 +36,7 @@ from nucliadb.common.context.fastapi import get_app_context
 from nucliadb.common.http_clients.processing import ProcessingHTTPClient
 from nucliadb.writer import logger
 from nucliadb.writer.settings import back_pressure_settings as settings
+from nucliadb_protos.writer_pb2 import ShardObject
 from nucliadb_telemetry import metrics
 from nucliadb_utils import const
 from nucliadb_utils.nats import NatsConnectionManager
@@ -112,9 +112,7 @@ def cached_back_pressure(kbid: str, resource_uuid: Optional[str] = None):
     if data is not None:
         try_after = data.try_after
         back_pressure_type = data.type
-        RATE_LIMITED_REQUESTS_COUNTER.inc(
-            {"type": back_pressure_type, "cached": "true"}
-        )
+        RATE_LIMITED_REQUESTS_COUNTER.inc({"type": back_pressure_type, "cached": "true"})
         logger.info(
             "Back pressure applied from cache",
             extra={
@@ -137,9 +135,7 @@ def cached_back_pressure(kbid: str, resource_uuid: Optional[str] = None):
     except BackPressureException as exc:
         try_after = exc.data.try_after
         back_pressure_type = exc.data.type
-        RATE_LIMITED_REQUESTS_COUNTER.inc(
-            {"type": back_pressure_type, "cached": "false"}
-        )
+        RATE_LIMITED_REQUESTS_COUNTER.inc({"type": back_pressure_type, "cached": "false"})
         _cache.set(cache_key, exc.data)
         raise HTTPException(
             status_code=429,
@@ -248,14 +244,10 @@ class Materializer:
                 for node in get_index_nodes():
                     try:
                         with back_pressure_observer({"type": "get_indexing_pending"}):
-                            self.indexing_pending[node.id] = (
-                                await get_nats_consumer_pending_messages(
-                                    self.nats_manager,
-                                    stream=const.Streams.INDEX.name,
-                                    consumer=const.Streams.INDEX.group.format(
-                                        node=node.id
-                                    ),
-                                )
+                            self.indexing_pending[node.id] = await get_nats_consumer_pending_messages(
+                                self.nats_manager,
+                                stream=const.Streams.INDEX.name,
+                                consumer=const.Streams.INDEX.group.format(node=node.id),
                             )
                     except Exception:
                         logger.exception(
@@ -336,9 +328,7 @@ def get_materializer() -> Materializer:
     return MATERIALIZER
 
 
-async def maybe_back_pressure(
-    request: Request, kbid: str, resource_uuid: Optional[str] = None
-) -> None:
+async def maybe_back_pressure(request: Request, kbid: str, resource_uuid: Optional[str] = None) -> None:
     """
     This function does system checks to see if we need to put back pressure on writes.
     In that case, a HTTP 429 will be raised with the estimated time to try again.
@@ -348,9 +338,7 @@ async def maybe_back_pressure(
     await back_pressure_checks(request, kbid, resource_uuid)
 
 
-async def back_pressure_checks(
-    request: Request, kbid: str, resource_uuid: Optional[str] = None
-):
+async def back_pressure_checks(request: Request, kbid: str, resource_uuid: Optional[str] = None):
     """
     Will raise a 429 if back pressure is needed:
     - If the processing engine is behind.
@@ -361,9 +349,7 @@ async def back_pressure_checks(
     materializer = get_materializer()
     with cached_back_pressure(kbid, resource_uuid):
         check_ingest_behind(materializer.get_ingest_pending())
-        await check_indexing_behind(
-            context, kbid, resource_uuid, materializer.get_indexing_pending()
-        )
+        await check_indexing_behind(context, kbid, resource_uuid, materializer.get_indexing_pending())
         await check_processing_behind(materializer, kbid)
 
 
@@ -418,9 +404,7 @@ async def check_indexing_behind(
 
     # Get nodes that are involved in the indexing of the request
     if resource_uuid is not None:
-        nodes_to_check = await get_nodes_for_resource_shard(
-            context, kbid, resource_uuid
-        )
+        nodes_to_check = await get_nodes_for_resource_shard(context, kbid, resource_uuid)
     else:
         nodes_to_check = await get_nodes_for_kb_active_shards(context, kbid)
 
@@ -488,9 +472,7 @@ def estimate_try_after(rate: float, pending: int, max_wait: int) -> datetime:
 
 
 @alru_cache(maxsize=1024, ttl=60 * 15)
-async def get_nodes_for_kb_active_shards(
-    context: ApplicationContext, kbid: str
-) -> list[str]:
+async def get_nodes_for_kb_active_shards(context: ApplicationContext, kbid: str) -> list[str]:
     with back_pressure_observer({"type": "get_kb_active_shard"}):
         active_shard = await get_kb_active_shard(context, kbid)
     if active_shard is None:
@@ -521,20 +503,16 @@ async def get_nats_consumer_pending_messages(
     return consumer_info.num_pending
 
 
-async def get_kb_active_shard(
-    context: ApplicationContext, kbid: str
-) -> Optional[ShardObject]:
-    async with context.kv_driver.transaction() as txn:
+async def get_kb_active_shard(context: ApplicationContext, kbid: str) -> Optional[ShardObject]:
+    async with context.kv_driver.transaction(read_only=True) as txn:
         return await context.shard_manager.get_current_active_shard(txn, kbid)
 
 
 async def get_resource_shard(
     context: ApplicationContext, kbid: str, resource_uuid: str
 ) -> Optional[ShardObject]:
-    async with datamanagers.with_transaction(read_only=True) as txn:
-        shard_id = await datamanagers.resources.get_resource_shard_id(
-            txn, kbid=kbid, rid=resource_uuid
-        )
+    async with datamanagers.with_ro_transaction() as txn:
+        shard_id = await datamanagers.resources.get_resource_shard_id(txn, kbid=kbid, rid=resource_uuid)
         if shard_id is None:
             # Resource does not exist
             logger.debug(

nucliadb/writer/exceptions.py

@@ -19,10 +19,6 @@
 #
 
 
-class ConflictError(Exception):
-    pass
-
-
 class ResourceNotFound(Exception):
     pass
 

nucliadb/writer/lifecycle.py

@@ -17,13 +17,20 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
-from nucliadb.ingest.processing import start_processing_engine
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from nucliadb.common.context.fastapi import inject_app_context
+from nucliadb.ingest.processing import start_processing_engine, stop_processing_engine
 from nucliadb.ingest.utils import start_ingest, stop_ingest
 from nucliadb.writer import SERVICE_NAME
+from nucliadb.writer.back_pressure import start_materializer, stop_materializer
+from nucliadb.writer.settings import back_pressure_settings
 from nucliadb.writer.tus import finalize as storage_finalize
 from nucliadb.writer.tus import initialize as storage_initialize
-from nucliadb.writer.utilities import get_processing
 from nucliadb_telemetry.utils import clean_telemetry, setup_telemetry
+from nucliadb_utils.settings import is_onprem_nucliadb
 from nucliadb_utils.utilities import (
     finalize_utilities,
     start_partitioning_utility,
@@ -32,29 +39,28 @@ from nucliadb_utils.utilities import (
 )
 
 
-async def initialize():
-    await setup_telemetry(SERVICE_NAME)
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    back_pressure_enabled = back_pressure_settings.enabled and not is_onprem_nucliadb()
 
+    await setup_telemetry(SERVICE_NAME)
     await start_ingest(SERVICE_NAME)
-
     await start_processing_engine()
-
     start_partitioning_utility()
-
     await start_transaction_utility(SERVICE_NAME)
     await storage_initialize()
 
+    # Inject application context into the fastapi app's state
+    async with inject_app_context(app) as context:
+        if back_pressure_enabled:
+            await start_materializer(context)
+        yield
 
-async def finalize():
+    if back_pressure_enabled:
+        await stop_materializer()
     await stop_transaction_utility()
-
     await stop_ingest()
-    processing = get_processing()
-    if processing is not None:
-        await processing.finalize()
-
+    await stop_processing_engine()
     await storage_finalize()
-
     await clean_telemetry(SERVICE_NAME)
-
     await finalize_utilities()

nucliadb/writer/resource/audit.py

@@ -19,9 +19,10 @@
 #
 from datetime import datetime
 
-from nucliadb_protos.writer_pb2 import Audit
 from starlette.requests import Request
 
+from nucliadb_protos.writer_pb2 import Audit
+
 
 def parse_audit(audit: Audit, request: Request):
     audit.user = request.headers.get("X-NUCLIADB-USER", "")

nucliadb/writer/resource/basic.py

@@ -18,8 +18,31 @@
 # along with this program. If not, see <http://www.gnu.org/licenses/>.
 #
 from datetime import datetime
+from typing import Optional
 
 from fastapi import HTTPException
+
+from nucliadb.common.models_utils import to_proto
+from nucliadb.common.models_utils.from_proto import (
+    RelationNodeTypeMap,
+    RelationTypeMap,
+)
+from nucliadb.ingest.orm.utils import set_title
+from nucliadb.ingest.processing import PushPayload
+from nucliadb_models.content_types import GENERIC_MIME_TYPE
+from nucliadb_models.file import FileField
+from nucliadb_models.link import LinkField
+from nucliadb_models.metadata import (
+    ParagraphAnnotation,
+    QuestionAnswerAnnotation,
+)
+from nucliadb_models.text import TEXT_FORMAT_TO_MIMETYPE, PushTextFormat, Text
+from nucliadb_models.writer import (
+    ComingResourcePayload,
+    CreateResourcePayload,
+    UpdateResourcePayload,
+)
+from nucliadb_protos.knowledgebox_pb2 import KnowledgeBoxConfig
 from nucliadb_protos.resources_pb2 import (
     Answers,
     Basic,
@@ -30,38 +53,19 @@ from nucliadb_protos.resources_pb2 import (
     Metadata,
     PageSelections,
     Paragraph,
+    TokenSplit,
+    UserFieldMetadata,
+    VisualSelection,
 )
 from nucliadb_protos.resources_pb2 import ParagraphAnnotation as PBParagraphAnnotation
 from nucliadb_protos.resources_pb2 import (
     QuestionAnswerAnnotation as PBQuestionAnswerAnnotation,
 )
-from nucliadb_protos.resources_pb2 import TokenSplit, UserFieldMetadata, VisualSelection
 from nucliadb_protos.utils_pb2 import Relation, RelationNode
 from nucliadb_protos.writer_pb2 import BrokerMessage
 
-from nucliadb.ingest.orm.utils import set_title
-from nucliadb.ingest.processing import PushPayload
-from nucliadb_models.common import FIELD_TYPES_MAP_REVERSE
-from nucliadb_models.file import FileField
-from nucliadb_models.link import LinkField
-from nucliadb_models.metadata import (
-    ParagraphAnnotation,
-    QuestionAnswerAnnotation,
-    RelationNodeTypeMap,
-    RelationTypeMap,
-)
-from nucliadb_models.text import TEXT_FORMAT_TO_MIMETYPE, PushTextFormat, Text
-from nucliadb_models.writer import (
-    GENERIC_MIME_TYPE,
-    ComingResourcePayload,
-    CreateResourcePayload,
-    UpdateResourcePayload,
-)
-
 
-def parse_basic_modify(
-    bm: BrokerMessage, item: ComingResourcePayload, toprocess: PushPayload
-):
+def parse_basic_modify(bm: BrokerMessage, item: ComingResourcePayload, toprocess: PushPayload):
     bm.basic.modified.FromDatetime(datetime.now())
     if item.title:
         set_title(bm, toprocess, item.title)
@@ -83,14 +87,9 @@ def parse_basic_modify(
         bm.basic.metadata.useful = True
         bm.basic.metadata.status = Metadata.Status.PENDING
 
-        toprocess.genericfield["summary"] = Text(
-            body=item.summary, format=PushTextFormat.PLAIN
-        )
+        toprocess.genericfield["summary"] = Text(body=item.summary, format=PushTextFormat.PLAIN)
     if item.thumbnail:
         bm.basic.thumbnail = item.thumbnail
-    if item.layout:
-        bm.basic.layout = item.layout
-
     if item.metadata is not None:
         bm.basic.metadata.metadata.update(item.metadata.metadata)
         if item.metadata.language:
@@ -147,9 +146,8 @@ def parse_basic_modify(
                 userfieldmetadata.question_answers.append(qa_annotation_pb)
 
             userfieldmetadata.field.field = fieldmetadata.field.field
-            userfieldmetadata.field.field_type = FIELD_TYPES_MAP_REVERSE[  # type: ignore
-                fieldmetadata.field.field_type.value
-            ]
+
+            userfieldmetadata.field.field_type = to_proto.field_type(fieldmetadata.field.field_type)
 
             bm.basic.fieldmetadata.append(userfieldmetadata)
 
@@ -167,9 +165,7 @@ def parse_basic_modify(
             ]
         )
 
-        relation_node_resource = RelationNode(
-            value=bm.uuid, ntype=RelationNode.NodeType.RESOURCE
-        )
+        relation_node_resource = RelationNode(value=bm.uuid, ntype=RelationNode.NodeType.RESOURCE)
         relations = []
         for relation in item.usermetadata.relations:
             if relation.from_ is None:
@@ -205,8 +201,16 @@ def parse_basic_modify(
         unique_groups = list(set(item.security.access_groups))
         bm.security.access_groups.extend(unique_groups)
 
+    if item.hidden is not None:
+        bm.basic.hidden = item.hidden
+
 
-def parse_basic(bm: BrokerMessage, item: CreateResourcePayload, toprocess: PushPayload):
+def parse_basic_creation(
+    bm: BrokerMessage,
+    item: CreateResourcePayload,
+    toprocess: PushPayload,
+    kb_config: Optional[KnowledgeBoxConfig],
+):
     bm.basic.created.FromDatetime(datetime.now())
 
     if item.title is None:
@@ -215,6 +219,10 @@ def parse_basic(bm: BrokerMessage, item: CreateResourcePayload, toprocess: PushP
 
     parse_basic_modify(bm, item, toprocess)
 
+    if item.hidden is None:
+        if kb_config and kb_config.hidden_resources_hide_on_creation:
+            bm.basic.hidden = True
+
 
 def set_status(basic: Basic, item: CreateResourcePayload):
     basic.metadata.status = Metadata.Status.PENDING
@@ -227,15 +235,11 @@ def set_status_modify(basic: Basic, item: UpdateResourcePayload):
 def validate_classifications(paragraph: ParagraphAnnotation):
     classifications = paragraph.classifications
     if len(classifications) == 0:
-        raise HTTPException(
-            status_code=422, detail="ensure classifications has at least 1 items"
-        )
+        raise HTTPException(status_code=422, detail="ensure classifications has at least 1 items")
 
-    unique_classifications = {tuple(cf.dict().values()) for cf in classifications}
+    unique_classifications = {tuple(cf.model_dump().values()) for cf in classifications}
     if len(unique_classifications) != len(classifications):
-        raise HTTPException(
-            status_code=422, detail="Paragraph classifications need to be unique"
-        )
+        raise HTTPException(status_code=422, detail="Paragraph classifications need to be unique")
 
 
 def compute_title(item: CreateResourcePayload, rid: str) -> str:
@@ -273,9 +277,7 @@ def build_question_answer_annotation_pb(
     pb.cancelled_by_user = qa_annotation.cancelled_by_user
     pb.question_answer.question.text = qa_annotation.question_answer.question.text
     if qa_annotation.question_answer.question.language is not None:
-        pb.question_answer.question.language = (
-            qa_annotation.question_answer.question.language
-        )
+        pb.question_answer.question.language = qa_annotation.question_answer.question.language
     pb.question_answer.question.ids_paragraphs.extend(
         qa_annotation.question_answer.question.ids_paragraphs
     )

nucliadb/writer/resource/field.py

@@ -21,23 +21,23 @@ from datetime import datetime
 from typing import Optional, Union
 
 from google.protobuf.json_format import MessageToDict
-from nucliadb_protos.writer_pb2 import BrokerMessage
 
 import nucliadb_models as models
+from nucliadb.common.models_utils import from_proto, to_proto
 from nucliadb.ingest.fields.conversation import Conversation
 from nucliadb.ingest.orm.resource import Resource as ORMResource
 from nucliadb.ingest.processing import PushPayload
 from nucliadb.writer import SERVICE_NAME
-from nucliadb.writer.layouts import serialize_blocks
 from nucliadb.writer.utilities import get_processing
-from nucliadb_models.common import FIELD_TYPES_MAP, FieldTypeName
+from nucliadb_models.common import FieldTypeName
+from nucliadb_models.content_types import GENERIC_MIME_TYPE
 from nucliadb_models.conversation import PushConversation
 from nucliadb_models.writer import (
-    GENERIC_MIME_TYPE,
     CreateResourcePayload,
     UpdateResourcePayload,
 )
 from nucliadb_protos import resources_pb2
+from nucliadb_protos.writer_pb2 import BrokerMessage
 from nucliadb_utils.storages.storage import StorageField
 from nucliadb_utils.utilities import get_storage
 
@@ -50,6 +50,7 @@ async def extract_file_field_from_pb(field_pb: resources_pb2.FieldFile) -> str:
             language=field_pb.language,
             password=field_pb.password,
             file=models.File(payload=None, uri=field_pb.file.uri),
+            extract_strategy=field_pb.extract_strategy,
         )
         return processing.convert_external_filefield_to_str(file_field)
     else:
@@ -80,12 +81,11 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
     storage = await get_storage(service_name=SERVICE_NAME)
     await resource.get_fields()
     for (field_type, field_id), field in resource.fields.items():
-        field_type_name = FIELD_TYPES_MAP[field_type]
+        field_type_name = from_proto.field_type_name(field_type)
 
         if field_type_name not in {
             FieldTypeName.TEXT,
             FieldTypeName.FILE,
-            FieldTypeName.LAYOUT,
             FieldTypeName.CONVERSATION,
             FieldTypeName.LINK,
         }:
@@ -114,28 +114,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
             parsed_text["format"] = models.PushTextFormat[parsed_text["format"]]
             toprocess.textfield[field_id] = models.Text(**parsed_text)
 
-        if field_type_name is FieldTypeName.LAYOUT:
-            parsed_layout = MessageToDict(
-                field_pb,
-                preserving_proto_field_name=True,
-                including_default_value_fields=True,
-            )
-            parsed_layout["format"] = resources_pb2.FieldLayout.Format.Value(
-                parsed_layout["format"]
-            )
-
-            for blockid, block in parsed_layout["body"]["blocks"].items():
-                cf = field_pb.body.blocks[blockid].file
-                block["file"] = await processing.convert_internal_cf_to_str(cf, storage)
-
-            parsed_layout["blocks"] = parsed_layout.get("body", {}).get("blocks", {})
-            del parsed_layout["body"]
-
-            toprocess.layoutfield[field_id] = models.LayoutDiff(**parsed_layout)
-
-        if field_type_name is FieldTypeName.CONVERSATION and isinstance(
-            field, Conversation
-        ):
+        if field_type_name is FieldTypeName.CONVERSATION and isinstance(field, Conversation):
             metadata = await field.get_metadata()
             if metadata.pages == 0:
                 continue
@@ -156,14 +135,13 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
                         await processing.convert_internal_cf_to_str(cf, storage)
                         for cf in message.content.attachments
                     ]
-                    parsed_message["content"]["format"] = (
-                        resources_pb2.MessageContent.Format.Value(
-                            parsed_message["content"]["format"]
-                        )
-                    )
-                    full_conversation.messages.append(
-                        models.PushMessage(**parsed_message)
+                    if "attachments_fields" in parsed_message["content"]:
+                        # Not defined on the push payload
+                        del parsed_message["content"]["attachments_fields"]
+                    parsed_message["content"]["format"] = resources_pb2.MessageContent.Format.Value(
+                        parsed_message["content"]["format"]
                     )
+                    full_conversation.messages.append(models.PushMessage(**parsed_message))
             toprocess.conversationfield[field_id] = full_conversation
 
 
@@ -176,9 +154,7 @@ async def parse_fields(
     x_skip_store: bool,
 ):
     for key, file_field in item.files.items():
-        await parse_file_field(
-            key, file_field, writer, toprocess, kbid, uuid, skip_store=x_skip_store
-        )
+        await parse_file_field(key, file_field, writer, toprocess, kbid, uuid, skip_store=x_skip_store)
 
     for key, link_field in item.links.items():
         parse_link_field(key, link_field, writer, toprocess)
@@ -186,19 +162,8 @@ async def parse_fields(
     for key, text_field in item.texts.items():
         parse_text_field(key, text_field, writer, toprocess)
 
-    for key, layout_field in item.layouts.items():
-        await parse_layout_field(key, layout_field, writer, toprocess, kbid, uuid)
-
     for key, conversation_field in item.conversations.items():
-        await parse_conversation_field(
-            key, conversation_field, writer, toprocess, kbid, uuid
-        )
-
-    for key, datetime_field in item.datetimes.items():
-        parse_datetime_field(key, datetime_field, writer, toprocess)
-
-    for key, keywordset_field in item.keywordsets.items():
-        parse_keywordset_field(key, keywordset_field, writer, toprocess)
+        await parse_conversation_field(key, conversation_field, writer, toprocess, kbid, uuid)
 
 
 def parse_text_field(
@@ -207,10 +172,10 @@ def parse_text_field(
     writer: BrokerMessage,
     toprocess: PushPayload,
 ) -> None:
+    if text_field.extract_strategy is not None:
+        writer.texts[key].extract_strategy = text_field.extract_strategy
     writer.texts[key].body = text_field.body
-    writer.texts[key].format = resources_pb2.FieldText.Format.Value(
-        text_field.format.value
-    )
+    writer.texts[key].format = resources_pb2.FieldText.Format.Value(text_field.format.value)
     etw = resources_pb2.ExtractedTextWrapper()
     etw.field.field = key
     etw.field.field_type = resources_pb2.FieldType.TEXT
@@ -219,6 +184,7 @@ def parse_text_field(
     toprocess.textfield[key] = models.Text(
         body=text_field.body,
         format=getattr(models.PushTextFormat, text_field.format.value),
+        extract_strategy=text_field.extract_strategy,
     )
 
 
@@ -251,6 +217,8 @@ async def parse_internal_file_field(
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
+    if file_field.extract_strategy is not None:
+        writer.files[key].extract_strategy = file_field.extract_strategy
 
     processing = get_processing()
 
@@ -286,6 +254,8 @@ def parse_external_file_field(
     writer.files[key].added.FromDatetime(datetime.now())
     if file_field.language:
         writer.files[key].language = file_field.language
+    if file_field.extract_strategy is not None:
+        writer.files[key].extract_strategy = file_field.extract_strategy
     uri = file_field.file.uri
     writer.files[key].url = uri  # type: ignore
     writer.files[key].file.uri = uri  # type: ignore
@@ -328,6 +298,9 @@ def parse_link_field(
     if link_field.xpath is not None:
         writer.links[key].xpath = link_field.xpath
 
+    if link_field.extract_strategy is not None:
+        writer.links[key].extract_strategy = link_field.extract_strategy
+
     toprocess.linkfield[key] = models.LinkUpload(
         link=link_field.uri,
         headers=link_field.headers or {},
@@ -335,78 +308,7 @@ def parse_link_field(
         localstorage=link_field.localstorage or {},
         css_selector=link_field.css_selector,
         xpath=link_field.xpath,
-    )
-
-
-def parse_keywordset_field(
-    key: str,
-    keywordset_field: models.FieldKeywordset,
-    writer: BrokerMessage,
-    toprocess: PushPayload,
-) -> None:
-    if keywordset_field.keywords is None:
-        return
-
-    for keyword in keywordset_field.keywords:
-        fieldpb = resources_pb2.Keyword()
-        fieldpb.value = keyword.value
-        writer.keywordsets[key].keywords.append(fieldpb)
-
-
-def parse_datetime_field(
-    key: str,
-    datetime_field: models.FieldDatetime,
-    writer: BrokerMessage,
-    toprocess: PushPayload,
-) -> None:
-    if datetime_field.value is None:
-        return
-
-    writer.datetimes[key].value.FromDatetime(datetime_field.value)
-
-
-async def parse_layout_field(
-    key: str,
-    layout_field: models.InputLayoutField,
-    writer: BrokerMessage,
-    toprocess: PushPayload,
-    kbid: str,
-    uuid: str,
-) -> None:
-    storage = await get_storage(service_name=SERVICE_NAME)
-    processing = get_processing()
-
-    lc: resources_pb2.FieldLayout = await serialize_blocks(
-        layout_field, kbid, uuid, key, storage
-    )
-    writer.layouts[key].CopyFrom(lc)
-
-    toprocess_blocks = {}
-    for blockid, block in layout_field.body.blocks.items():
-        sf_conv_field: StorageField = storage.layout_field(
-            kbid, uuid, field=key, ident=block.ident
-        )
-        cf_conv_field = await storage.upload_b64file_to_cloudfile(
-            sf_conv_field,
-            block.file.payload.encode(),
-            block.file.filename,
-            block.file.content_type,
-            block.file.md5,
-        )
-
-        toprocess_blocks[blockid] = models.PushLayoutBlock(
-            x=block.x,
-            y=block.y,
-            cols=block.cols,
-            rows=block.rows,
-            type=block.type,
-            ident=block.ident,
-            payload=block.payload,
-            file=await processing.convert_internal_cf_to_str(cf_conv_field, storage),
-        )
-
-    toprocess.layoutfield[key] = models.LayoutDiff(
-        format=lc.format, blocks=toprocess_blocks  # type: ignore
+        extract_strategy=link_field.extract_strategy,
     )
 
 
@@ -420,7 +322,6 @@ async def parse_conversation_field(
 ) -> None:
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
-
     field_value = resources_pb2.Conversation()
     convs = models.PushConversation()
     for message in conversation_field.messages:
@@ -441,8 +342,16 @@ async def parse_conversation_field(
         )
 
         cm.content.text = message.content.text
-        cm.content.format = resources_pb2.MessageContent.Format.Value(
-            message.content.format.value
+        cm.content.format = resources_pb2.MessageContent.Format.Value(message.content.format.value)
+        cm.content.attachments_fields.extend(
+            [
+                resources_pb2.FieldRef(
+                    field_type=to_proto.field_type_name(attachment.field_type),
+                    field_id=attachment.field_id,
+                    split=attachment.split if attachment.split is not None else "",
+                )
+                for attachment in message.content.attachments_fields
+            ]
         )
 
         for count, file in enumerate(message.content.attachments):