nucliadb 6.3.5.post3990__py3-none-any.whl → 6.3.5.post3996__py3-none-any.whl

nucliadb/ingest/orm/brain_v2.py

@@ -24,6 +24,7 @@ from typing import Optional
 
 from nucliadb.common import ids
 from nucliadb.ingest import logger
+from nucliadb.ingest.orm.metrics import brain_observer as observer
 from nucliadb.ingest.orm.utils import compute_paragraph_key
 from nucliadb_models.labels import BASE_LABELS, LABEL_HIDDEN, flatten_resource_labels
 from nucliadb_models.metadata import ResourceProcessingStatus
@@ -74,7 +75,8 @@ class ResourceBrainV2:
         self.brain: PBBrainResource = PBBrainResource(resource=ResourceID(uuid=rid))
         self.labels: dict[str, set[str]] = deepcopy(BASE_LABELS)
 
-    def generate_resource_indexing_metadata(
+    @observer.wrap({"type": "generate_resource_metadata"})
+    def generate_resource_metadata(
         self,
         basic: Basic,
         user_relations: Relations,
@@ -89,7 +91,8 @@ class ResourceBrainV2:
         if security is not None:
             self._set_resource_security(security)
 
-    def generate_texts_index_message(
+    @observer.wrap({"type": "generate_texts"})
+    def generate_texts(
         self,
         field_key: str,
         extracted_text: ExtractedText,
@@ -112,6 +115,7 @@ class ResourceBrainV2:
             basic_user_metadata,
         )
 
+    @observer.wrap({"type": "apply_field_text"})
     def apply_field_text(
         self,
         field_key: str,
@@ -131,6 +135,7 @@ class ResourceBrainV2:
             full_field_id = ids.FieldId(rid=self.rid, type=ftype, key=fkey).full()
             self.brain.texts_to_delete.append(full_field_id)
 
+    @observer.wrap({"type": "apply_field_labels"})
     def apply_field_labels(
         self,
         field_key: str,
@@ -200,7 +205,8 @@ class ResourceBrainV2:
 
         self.brain.texts[field_key].labels.extend(flatten_resource_labels(labels))
 
-    def generate_paragraphs_index_message(
+    @observer.wrap({"type": "generate_paragraphs"})
+    def generate_paragraphs(
         self,
         field_key: str,
         field_computed_metadata: FieldComputedMetadata,
@@ -228,6 +234,7 @@ class ResourceBrainV2:
             skip_paragraphs=skip_index,
         )
 
+    @observer.wrap({"type": "apply_field_paragraphs"})
     def apply_field_paragraphs(
         self,
         field_key: str,
@@ -371,7 +378,8 @@ class ResourceBrainV2:
                     pc.valid.setdefault(paragraph_key, []).append(classif_label)
         return pc
 
-    def generate_relations_index_message(
+    @observer.wrap({"type": "generate_relations"})
+    def generate_relations(
         self,
         field_key: str,
         field_computed_metadata: Optional[FieldComputedMetadata],
@@ -477,7 +485,8 @@ class ResourceBrainV2:
         self.brain.sentences_to_delete.append(full_field_id)
         self.brain.relation_fields_to_delete.append(field_key)
 
-    def generate_vectors_index_message(
+    @observer.wrap({"type": "generate_vectors"})
+    def generate_vectors(
         self,
         field_id: str,
         vo: utils_pb2.VectorObject,
@@ -547,6 +556,7 @@ class ResourceBrainV2:
             full_field_id = ids.FieldId(rid=self.rid, type=fid.type, key=fid.key).full()
             self.brain.vector_prefixes_to_delete[vectorset].items.append(full_field_id)
 
+    @observer.wrap({"type": "apply_field_vector"})
     def _apply_field_vector(
         self,
         field_id: str,
@@ -764,6 +774,7 @@ class ParagraphPages:
         self.positions = positions
         self._materialized = self._materialize_page_numbers(positions)
 
+    @observer.wrap({"type": "materialize_page_numbers"})
     def _materialize_page_numbers(self, positions: FilePagePositions) -> list[int]:
         page_numbers_by_index = []
         for page_number, (page_start, page_end) in positions.items():

nucliadb/ingest/orm/index_message.py

@@ -26,6 +26,7 @@ from nucliadb.common import datamanagers
 from nucliadb.ingest.fields.exceptions import FieldAuthorNotFound
 from nucliadb.ingest.fields.file import File
 from nucliadb.ingest.orm.brain_v2 import ResourceBrainV2 as ResourceBrain
+from nucliadb.ingest.orm.metrics import index_message_observer as observer
 from nucliadb.ingest.orm.resource import Resource, get_file_page_positions
 from nucliadb_protos.knowledgebox_pb2 import VectorSetConfig
 from nucliadb_protos.noderesources_pb2 import Resource as IndexMessage
@@ -40,6 +41,7 @@ class IndexMessageBuilder:
         self.resource = resource
         self.brain = ResourceBrain(resource.uuid)
 
+    @observer.wrap({"type": "resource_data"})
     async def _apply_resource_index_data(self, brain: ResourceBrain) -> None:
         # Set the metadata at the resource level
         basic = await self.resource.get_basic()
@@ -48,7 +50,7 @@ class IndexMessageBuilder:
         origin = await self.resource.get_origin()
         security = await self.resource.get_security()
         await asyncio.to_thread(
-            brain.generate_resource_indexing_metadata,
+            brain.generate_resource_metadata,
             basic,
             user_relations,
             origin,
@@ -56,6 +58,7 @@ class IndexMessageBuilder:
             security,
         )
 
+    @observer.wrap({"type": "field_data"})
     async def _apply_field_index_data(
         self,
         brain: ResourceBrain,
@@ -87,7 +90,7 @@ class IndexMessageBuilder:
                 except FieldAuthorNotFound:
                     field_author = None
                 await asyncio.to_thread(
-                    brain.generate_texts_index_message,
+                    brain.generate_texts,
                     self.resource.generate_field_id(fieldid),
                     extracted_text,
                     field_computed_metadata,
@@ -108,7 +111,7 @@ class IndexMessageBuilder:
                     await get_file_page_positions(field) if isinstance(field, File) else None
                 )
                 await asyncio.to_thread(
-                    brain.generate_paragraphs_index_message,
+                    brain.generate_paragraphs,
                     self.resource.generate_field_id(fieldid),
                     field_computed_metadata,
                     extracted_text,
@@ -127,7 +130,7 @@ class IndexMessageBuilder:
                 if vo is not None:
                     dimension = vectorset_config.vectorset_index_config.vector_dimension
                     await asyncio.to_thread(
-                        brain.generate_vectors_index_message,
+                        brain.generate_vectors,
                         self.resource.generate_field_id(fieldid),
                         vo,
                         vectorset=vectorset_config.vectorset_id,
@@ -136,7 +139,7 @@ class IndexMessageBuilder:
                     )
         if relations:
             await asyncio.to_thread(
-                brain.generate_relations_index_message,
+                brain.generate_relations,
                 self.resource.generate_field_id(fieldid),
                 field_computed_metadata,
                 basic.usermetadata,
@@ -151,6 +154,7 @@ class IndexMessageBuilder:
         for field_id in field_ids:
             brain.delete_field(self.resource.generate_field_id(field_id))
 
+    @observer.wrap({"type": "writer_bm"})
     async def for_writer_bm(
         self,
         messages: list[BrokerMessage],
@@ -192,6 +196,7 @@ class IndexMessageBuilder:
             )
         return self.brain.brain
 
+    @observer.wrap({"type": "processor_bm"})
     async def for_processor_bm(
         self,
         messages: list[BrokerMessage],
@@ -223,6 +228,7 @@ class IndexMessageBuilder:
             )
         return self.brain.brain
 
+    @observer.wrap({"type": "full"})
     async def full(self, reindex: bool) -> IndexMessage:
         await self._apply_resource_index_data(self.brain)
         basic = await self.get_basic()

nucliadb/ingest/orm/metrics.py

@@ -22,6 +22,17 @@ from nucliadb_telemetry import metrics
 
 processor_observer = metrics.Observer(
     "nucliadb_ingest_processor",
-    labels={"type": "", "source": ""},
+    labels={"type": ""},
     error_mappings={"kb_conflict": KnowledgeBoxConflict},
 )
+
+
+index_message_observer = metrics.Observer(
+    "index_message_builder",
+    labels={"type": ""},
+)
+
+brain_observer = metrics.Observer(
+    "brain",
+    labels={"type": ""},
+)

nucliadb/ingest/orm/processor/data_augmentation.py

@@ -23,8 +23,8 @@ from dataclasses import dataclass, field
 from typing import Optional
 
 from nucliadb.ingest.orm.resource import Resource
-from nucliadb.ingest.processing import ProcessingEngine, PushPayload, Source
-from nucliadb_models.text import PushTextFormat, Text
+from nucliadb.ingest.processing import ProcessingEngine
+from nucliadb.models.internal.processing import PushPayload, PushTextFormat, Source, Text
 from nucliadb_protos import resources_pb2, writer_pb2
 from nucliadb_protos.resources_pb2 import FieldType
 from nucliadb_utils.utilities import Utility, get_partitioning, get_utility

nucliadb/ingest/orm/processor/processor.py

@@ -462,6 +462,7 @@ class Processor:
                 source=source,
             )
 
+    @processor_observer.wrap({"type": "generate_index_message_v2"})
     async def generate_index_message_v2(
         self,
         resource: Resource,
@@ -471,14 +472,13 @@ class Processor:
         builder = IndexMessageBuilder(resource)
         message_source = messages_source(messages)
         if message_source == nodewriter_pb2.IndexMessageSource.WRITER:
-            with processor_observer({"type": "generate_index_message", "source": "writer"}):
-                return await builder.for_writer_bm(messages, resource_created)
+            return await builder.for_writer_bm(messages, resource_created)
         elif message_source == nodewriter_pb2.IndexMessageSource.PROCESSOR:
-            with processor_observer({"type": "generate_index_message", "source": "processor"}):
-                return await builder.for_processor_bm(messages)
+            return await builder.for_processor_bm(messages)
         else:  # pragma: no cover
             raise InvalidBrokerMessage(f"Unknown broker message source: {message_source}")
 
+    @processor_observer.wrap({"type": "generate_index_message_v1"})
     async def generate_index_message_v1(
         self,
         resource: Resource,

nucliadb/ingest/orm/utils.py

@@ -20,8 +20,7 @@
 import urllib.parse
 from typing import Sequence
 
-from nucliadb.ingest.processing import PushPayload
-from nucliadb_models.text import PushTextFormat, Text
+from nucliadb.models.internal.processing import PushPayload, PushTextFormat, Text
 from nucliadb_protos.resources_pb2 import (
     ExtractedTextWrapper,
     FieldComputedMetadataWrapper,

nucliadb/ingest/processing.py

@@ -25,15 +25,14 @@ import uuid
 from collections import defaultdict
 from contextlib import AsyncExitStack
 from enum import Enum
-from typing import TYPE_CHECKING, Any, Optional, TypeVar
+from typing import Any, Optional, TypeVar
 
 import aiohttp
 import backoff
 import jwt
-from pydantic import BaseModel, Field
 
 import nucliadb_models as models
-from nucliadb_models.labels import ClassificationLabel
+from nucliadb.models.internal.processing import ClassificationLabel, ProcessingInfo, PushPayload
 from nucliadb_models.resource import QueueType
 from nucliadb_protos.resources_pb2 import CloudFile
 from nucliadb_protos.resources_pb2 import FieldFile as FieldFilePB
@@ -52,10 +51,6 @@ logger = logging.getLogger(__name__)
 
 _T = TypeVar("_T")
 
-if TYPE_CHECKING:  # pragma: no cover
-    SourceValue = CloudFile.Source.V
-else:
-    SourceValue = int
 
 RETRIABLE_EXCEPTIONS = (aiohttp.client_exceptions.ClientConnectorError,)
 MAX_TRIES = 4
@@ -71,53 +66,6 @@ processing_observer = metrics.Observer(
 )
 
 
-class Source(SourceValue, Enum):  # type: ignore
-    HTTP = 0
-    INGEST = 1
-
-
-class ProcessingInfo(BaseModel):
-    seqid: Optional[int] = None
-    account_seq: Optional[int] = None
-    queue: Optional[QueueType] = None
-
-
-class PushPayload(BaseModel):
-    # There are multiple options of payload
-    uuid: str
-    slug: Optional[str] = None
-    kbid: str
-    source: Optional[Source] = None
-    userid: str
-
-    title: Optional[str] = None
-
-    genericfield: dict[str, models.Text] = {}
-
-    # New File
-    filefield: dict[str, str] = Field(
-        default={},
-        description="Map of each file field to the jwt token computed in ProcessingEngine methods",
-    )
-
-    # New Link
-    linkfield: dict[str, models.LinkUpload] = {}
-
-    # Diff on Text Field
-    textfield: dict[str, models.Text] = {}
-
-    # New conversations to process
-    conversationfield: dict[str, models.PushConversation] = {}
-
-    # Only internal
-    partition: int
-
-    # List of available processing options (with default values)
-    processing_options: Optional[models.PushProcessingOptions] = Field(
-        default_factory=models.PushProcessingOptions
-    )
-
-
 async def start_processing_engine():
     processing_engine = get_utility(Utility.PROCESSING)
     if processing_engine is not None:

nucliadb/models/internal/__init__.py

@@ -0,0 +1,19 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#

nucliadb/models/internal/processing.py

@@ -0,0 +1,160 @@
+# Copyright (C) 2021 Bosutech XXI S.L.
+#
+# nucliadb is offered under the AGPL v3.0 and as commercial software.
+# For commercial licensing, contact us at info@nuclia.com.
+#
+# AGPL:
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+
+# Processing classes (Those used to sent to push endpoints)
+
+
+from datetime import datetime
+from enum import Enum
+from typing import TYPE_CHECKING, Optional
+
+from pydantic import BaseModel, Field
+
+from nucliadb_models.processing import PushProcessingOptions
+from nucliadb_models.resource import QueueType
+from nucliadb_protos.resources_pb2 import CloudFile
+
+if TYPE_CHECKING:  # pragma: no cover
+    SourceValue = CloudFile.Source.V
+else:
+    SourceValue = int
+
+
+class ClassificationLabel(BaseModel):
+    """
+    NOTE: This model is used to send the labels of each field in the processing requests.
+    It is a rath is not meant to be used by api users.
+    """
+
+    labelset: str
+    label: str
+
+    def __hash__(self):
+        return hash((self.labelset, self.label))
+
+
+class PushTextFormat(int, Enum):
+    PLAIN = 0
+    HTML = 1
+    MARKDOWN = 2
+    RST = 3
+    JSON = 4
+    KEEP_MARKDOWN = 5
+    JSONL = 6
+    PLAIN_BLANKLINE_SPLIT = 7
+
+
+class Text(BaseModel):
+    body: str
+    format: PushTextFormat
+    extract_strategy: Optional[str] = None
+    classification_labels: list[ClassificationLabel] = []
+
+
+class LinkUpload(BaseModel):
+    link: str
+    headers: dict[str, str] = {}
+    cookies: dict[str, str] = {}
+    localstorage: dict[str, str] = {}
+    css_selector: Optional[str] = Field(
+        None,
+        title="Css selector",
+        description="Css selector to parse the link",
+    )
+    xpath: Optional[str] = Field(
+        None,
+        title="Xpath",
+        description="Xpath to parse the link",
+    )
+    extract_strategy: Optional[str] = None
+    classification_labels: list[ClassificationLabel] = []
+
+
+class PushMessageFormat(int, Enum):
+    PLAIN = 0
+    HTML = 1
+    MARKDOWN = 2
+    RST = 3
+    JSON = 4
+
+
+class PushMessageContent(BaseModel):
+    text: Optional[str] = None
+    format: PushMessageFormat
+    attachments: list[str] = []
+
+
+class PushMessage(BaseModel):
+    timestamp: Optional[datetime] = None
+    who: Optional[str] = None
+    to: list[str] = []
+    content: PushMessageContent
+    ident: str
+
+
+class PushConversation(BaseModel):
+    messages: list[PushMessage] = []
+    extract_strategy: Optional[str] = None
+    classification_labels: list[ClassificationLabel] = []
+
+
+class Source(SourceValue, Enum):  # type: ignore
+    HTTP = 0
+    INGEST = 1
+
+
+class ProcessingInfo(BaseModel):
+    seqid: Optional[int] = None
+    account_seq: Optional[int] = None
+    queue: Optional[QueueType] = None
+
+
+class PushPayload(BaseModel):
+    uuid: str
+    slug: Optional[str] = None
+    kbid: str
+    source: Optional[Source] = None
+    userid: str
+
+    title: Optional[str] = None
+
+    genericfield: dict[str, Text] = {}
+
+    # New File
+    filefield: dict[str, str] = Field(
+        default={},
+        description="Map of each file field to the jwt token computed in ProcessingEngine methods",
+    )
+
+    # New Link
+    linkfield: dict[str, LinkUpload] = {}
+
+    # Diff on Text Field
+    textfield: dict[str, Text] = {}
+
+    # New conversations to process
+    conversationfield: dict[str, PushConversation] = {}
+
+    # Only internal
+    partition: int
+
+    # List of available processing options (with default values)
+    processing_options: Optional[PushProcessingOptions] = Field(default_factory=PushProcessingOptions)

nucliadb/writer/api/v1/field.py

@@ -27,7 +27,7 @@ from starlette.requests import Request
 import nucliadb_models as models
 from nucliadb.common.maindb.utils import get_driver
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
-from nucliadb.ingest.processing import PushPayload, Source
+from nucliadb.models.internal.processing import PushPayload, Source
 from nucliadb.writer import SERVICE_NAME
 from nucliadb.writer.api.constants import (
     X_FILE_PASSWORD,

nucliadb/writer/api/v1/resource.py

@@ -33,7 +33,7 @@ from nucliadb.common.maindb.driver import Driver
 from nucliadb.common.maindb.exceptions import ConflictError, NotFoundError
 from nucliadb.common.maindb.utils import get_driver
 from nucliadb.ingest.orm.knowledgebox import KnowledgeBox
-from nucliadb.ingest.processing import ProcessingInfo, PushPayload, Source
+from nucliadb.models.internal.processing import ProcessingInfo, PushPayload, Source
 from nucliadb.writer import SERVICE_NAME, logger
 from nucliadb.writer.api.constants import X_NUCLIADB_USER, X_SKIP_STORE
 from nucliadb.writer.api.v1 import transaction

nucliadb/writer/api/v1/upload.py

@@ -33,7 +33,7 @@ from starlette.requests import Request as StarletteRequest
 
 from nucliadb.common import datamanagers
 from nucliadb.ingest.orm.utils import set_title
-from nucliadb.ingest.processing import PushPayload, Source
+from nucliadb.models.internal.processing import PushPayload, Source
 from nucliadb.models.responses import HTTPClientError
 from nucliadb.writer import SERVICE_NAME
 from nucliadb.writer.api.constants import X_EXTRACT_STRATEGY, X_FILENAME, X_LANGUAGE, X_MD5, X_PASSWORD

nucliadb/writer/resource/basic.py

@@ -28,16 +28,15 @@ from nucliadb.common.models_utils.from_proto import (
     RelationTypeMap,
 )
 from nucliadb.ingest.orm.utils import set_title
-from nucliadb.ingest.processing import PushPayload
+from nucliadb.models.internal.processing import ClassificationLabel, PushPayload, PushTextFormat, Text
 from nucliadb_models.content_types import GENERIC_MIME_TYPE
 from nucliadb_models.file import FileField
-from nucliadb_models.labels import ClassificationLabel
 from nucliadb_models.link import LinkField
 from nucliadb_models.metadata import (
     ParagraphAnnotation,
     QuestionAnswerAnnotation,
 )
-from nucliadb_models.text import TEXT_FORMAT_TO_MIMETYPE, PushTextFormat, Text
+from nucliadb_models.text import TEXT_FORMAT_TO_MIMETYPE
 from nucliadb_models.writer import (
     ComingResourcePayload,
     CreateResourcePayload,

nucliadb/writer/resource/field.py

@@ -29,13 +29,12 @@ from nucliadb.common.maindb.driver import Transaction
 from nucliadb.common.models_utils import from_proto, to_proto
 from nucliadb.ingest.fields.conversation import Conversation
 from nucliadb.ingest.orm.resource import Resource as ORMResource
-from nucliadb.ingest.processing import PushPayload
+from nucliadb.models.internal import processing as processing_models
+from nucliadb.models.internal.processing import ClassificationLabel, PushConversation, PushPayload
 from nucliadb.writer import SERVICE_NAME
 from nucliadb.writer.utilities import get_processing
 from nucliadb_models.common import FieldTypeName
 from nucliadb_models.content_types import GENERIC_MIME_TYPE
-from nucliadb_models.conversation import PushConversation
-from nucliadb_models.labels import ClassificationLabel
 from nucliadb_models.writer import (
     CreateResourcePayload,
     UpdateResourcePayload,
@@ -134,7 +133,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
                 including_default_value_fields=True,
             )
             parsed_link["link"] = parsed_link.pop("uri", None)
-            toprocess.linkfield[field_id] = models.LinkUpload(**parsed_link)
+            toprocess.linkfield[field_id] = processing_models.LinkUpload(**parsed_link)
             toprocess.linkfield[field_id].classification_labels = classif_labels
 
         if field_type_name is FieldTypeName.TEXT:
@@ -143,8 +142,8 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
                 preserving_proto_field_name=True,
                 including_default_value_fields=True,
             )
-            parsed_text["format"] = models.PushTextFormat[parsed_text["format"]]
-            toprocess.textfield[field_id] = models.Text(**parsed_text)
+            parsed_text["format"] = processing_models.PushTextFormat[parsed_text["format"]]
+            toprocess.textfield[field_id] = processing_models.Text(**parsed_text)
             toprocess.textfield[field_id].classification_labels = classif_labels
 
         if field_type_name is FieldTypeName.CONVERSATION and isinstance(field, Conversation):
@@ -174,7 +173,7 @@ async def extract_fields(resource: ORMResource, toprocess: PushPayload):
                     parsed_message["content"]["format"] = resources_pb2.MessageContent.Format.Value(
                         parsed_message["content"]["format"]
                     )
-                    full_conversation.messages.append(models.PushMessage(**parsed_message))
+                    full_conversation.messages.append(processing_models.PushMessage(**parsed_message))
             toprocess.conversationfield[field_id] = full_conversation
             toprocess.conversationfield[field_id].classification_labels = classif_labels
 
@@ -247,9 +246,9 @@ def parse_text_field(
     etw.field.field_type = resources_pb2.FieldType.TEXT
     etw.body.text = text_field.body
     writer.extracted_text.append(etw)
-    toprocess.textfield[key] = models.Text(
+    toprocess.textfield[key] = processing_models.Text(
         body=text_field.body,
-        format=getattr(models.PushTextFormat, text_field.format.value),
+        format=getattr(processing_models.PushTextFormat, text_field.format.value),
         extract_strategy=text_field.extract_strategy,
         classification_labels=classif_labels,
     )
@@ -393,7 +392,7 @@ def parse_link_field(
     if link_field.extract_strategy is not None:
         writer.links[key].extract_strategy = link_field.extract_strategy
 
-    toprocess.linkfield[key] = models.LinkUpload(
+    toprocess.linkfield[key] = processing_models.LinkUpload(
         link=link_field.uri,
         headers=link_field.headers or {},
         cookies=link_field.cookies or {},
@@ -424,7 +423,7 @@ async def parse_conversation_field(
     storage = await get_storage(service_name=SERVICE_NAME)
     processing = get_processing()
     field_value = resources_pb2.Conversation()
-    convs = models.PushConversation()
+    convs = processing_models.PushConversation()
     for message in conversation_field.messages:
         cm = resources_pb2.Message()
         if message.timestamp:
@@ -437,9 +436,9 @@ async def parse_conversation_field(
         if message.type_ is not None:
             cm.type = resources_pb2.Message.MessageType.Value(message.type_.value)
 
-        processing_message_content = models.PushMessageContent(
+        processing_message_content = processing_models.PushMessageContent(
             text=message.content.text,
-            format=getattr(models.PushMessageFormat, message.content.format.value),
+            format=getattr(processing_models.PushMessageFormat, message.content.format.value),
         )
 
         cm.content.text = message.content.text
@@ -472,7 +471,7 @@ async def parse_conversation_field(
                 await processing.convert_internal_cf_to_str(cf_conv_field, storage)
             )
 
-        processing_message = models.PushMessage(
+        processing_message = processing_models.PushMessage(
             timestamp=message.timestamp,
             content=processing_message_content,
             ident=message.ident,

{nucliadb-6.3.5.post3990.dist-info → nucliadb-6.3.5.post3996.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb
-Version: 6.3.5.post3990
+Version: 6.3.5.post3996
 Summary: NucliaDB
 Author-email: Nuclia <nucliadb@nuclia.com>
 License: AGPL
@@ -20,11 +20,11 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Requires-Python: <4,>=3.9
 Description-Content-Type: text/markdown
-Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3990
-Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3990
-Requires-Dist: nucliadb-protos>=6.3.5.post3990
-Requires-Dist: nucliadb-models>=6.3.5.post3990
-Requires-Dist: nidx-protos>=6.3.5.post3990
+Requires-Dist: nucliadb-telemetry[all]>=6.3.5.post3996
+Requires-Dist: nucliadb-utils[cache,fastapi,storages]>=6.3.5.post3996
+Requires-Dist: nucliadb-protos>=6.3.5.post3996
+Requires-Dist: nucliadb-models>=6.3.5.post3996
+Requires-Dist: nidx-protos>=6.3.5.post3996
 Requires-Dist: nucliadb-admin-assets>=1.0.0.post1224
 Requires-Dist: nuclia-models>=0.24.2
 Requires-Dist: uvicorn[standard]

{nucliadb-6.3.5.post3990.dist-info → nucliadb-6.3.5.post3996.dist-info}/RECORD

@@ -118,7 +118,7 @@ nucliadb/export_import/utils.py,sha256=8XOVMYXXw8b4ikojG7RjQ4tKN3Xu7nfu2yCUOqD50
 nucliadb/ingest/__init__.py,sha256=fsw3C38VP50km3R-nHL775LNGPpJ4JxqXJ2Ib1f5SqE,1011
 nucliadb/ingest/app.py,sha256=TaVgh5B2riFVmcsrbPb7a5YCzmnybjx-NK0BXgTwGAY,7535
 nucliadb/ingest/partitions.py,sha256=2NIhMYbNT0TNBL6bX1UMSi7vxFGICstCKEqsB0TXHOE,2410
-nucliadb/ingest/processing.py,sha256=7NNoVxbSwsRdbo5goqVSrUc_QXZRVfOT_jZPzrmbxJQ,22207
+nucliadb/ingest/processing.py,sha256=QmkHq-BU4vub7JRWe9VHvQ2DcAmT6-CzgFXuZxXhcBU,20953
 nucliadb/ingest/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb/ingest/serialize.py,sha256=-TIjibJTbMqAowzRvyrG3R209vKqBZqXpdrQL9Dq4lo,16135
 nucliadb/ingest/settings.py,sha256=0B-wQNa8FLqtNcQgRzh-fuIuGptM816XHcbH1NQKfmE,3050
@@ -142,20 +142,20 @@ nucliadb/ingest/fields/link.py,sha256=kN_gjRUEEj5cy8K_BwPijYg3TiWhedc24apXYlTbRJ
 nucliadb/ingest/fields/text.py,sha256=tFvSQJAe0W7ePpp2_WDfLiE2yglR1OTU0Zht9acvOFw,1594
 nucliadb/ingest/orm/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/ingest/orm/brain.py,sha256=8nXdxgI3zYn6DGnCq5ciq3PA7ouhcTW5dSgHaxAO6xg,29074
-nucliadb/ingest/orm/brain_v2.py,sha256=3ejtH58X9Hkhvg2m7wrp2eEyIQybKSMHzoJBDtkL0b8,33065
+nucliadb/ingest/orm/brain_v2.py,sha256=XEOfvjpnvSKNrAOtbO4vt9n_PWVbzOhB-seHs76uY0M,33588
 nucliadb/ingest/orm/broker_message.py,sha256=XWaiZgDOz94NPOPT-hqbRr5ZkpVimUw6PjUJNftfoVw,7514
 nucliadb/ingest/orm/entities.py,sha256=a-aYuKBUQhxDKFtXOzTAkLlY_t2JiTfaptw2vt3AQDQ,14915
 nucliadb/ingest/orm/exceptions.py,sha256=k4Esv4NtL4TrGTcsQpwrSfDhPQpiYcRbB1SpYmBX5MY,1432
-nucliadb/ingest/orm/index_message.py,sha256=3Okq-POmGnxpgzysU1bvgj5skRLSDBUpP6dnTYqNA18,15821
+nucliadb/ingest/orm/index_message.py,sha256=fFNYRZTH45fm6IZ9tHNwa4KNgV8KxzwS5uuklRe65ww,16044
 nucliadb/ingest/orm/knowledgebox.py,sha256=Bfb4-MIQWlaJrQAUDbgs_iIsXCYjS7s5YiiGl_Jb4jo,23887
-nucliadb/ingest/orm/metrics.py,sha256=z-xVOJWeWXZFPIzRg_NB8nlbkdJFs_myEwLROdTNp24,1110
+nucliadb/ingest/orm/metrics.py,sha256=OiuggTh-n3kZHA2G73NEUdIlh8c3yFrbusI88DK-Mko,1273
 nucliadb/ingest/orm/resource.py,sha256=GjxcEPuu8bM06Uea7_yJk0UFvOfiZNP9i_G4V-4D8_U,46845
-nucliadb/ingest/orm/utils.py,sha256=vCe_9UxHu26JDFGLwQ0wH-XyzJIpQCTK-Ow9dtZR5Vg,2716
+nucliadb/ingest/orm/utils.py,sha256=fCQRuyecgqhaY7mcBG93oaXMkzkKb9BFjOcy4-ZiSNw,2693
 nucliadb/ingest/orm/processor/__init__.py,sha256=Aqd9wCNTvggkMkCY3WvoI8spdr94Jnqk-0iq9XpLs18,922
 nucliadb/ingest/orm/processor/auditing.py,sha256=TeYhXGJRyQ7ROytbb2u8R0fIh_FYi3HgTu3S1ribY3U,4623
-nucliadb/ingest/orm/processor/data_augmentation.py,sha256=HpSU9olDHcTfECDYCsmm4yA-Hu0mBrd_zTtx50XDGFE,5164
+nucliadb/ingest/orm/processor/data_augmentation.py,sha256=v-pj4GbBWSuO8dQyahs5UDr5ghsyfhCZDS0ftKd6ZYc,5179
 nucliadb/ingest/orm/processor/pgcatalog.py,sha256=H-OCRz0RuTUb80LZBxDowLA9V7ECv1DWiXlnzKW5XGI,3103
-nucliadb/ingest/orm/processor/processor.py,sha256=flw2U1OB6il9mP7h6fCY_dZX-Jo6XemWq1dAwtH7pMs,33202
+nucliadb/ingest/orm/processor/processor.py,sha256=q2iBJJ_5SV_bxA3t5MrbV70iQhir94aFbjZjnYJzEAQ,33141
 nucliadb/ingest/orm/processor/sequence_manager.py,sha256=uqEphtI1Ir_yk9jRl2gPf7BlzzXWovbARY5MNZSBI_8,1704
 nucliadb/ingest/service/__init__.py,sha256=MME_G_ERxzJR6JW_hfE2qcfXpmpH1kdG-S0a-M0qRm8,2043
 nucliadb/ingest/service/exceptions.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
@@ -172,6 +172,8 @@ nucliadb/migrator/settings.py,sha256=3eK6PIwqqtoCea9V7-RMjVfZC_0BC5DqPPlfo9XMyNE
 nucliadb/migrator/utils.py,sha256=NgUreUvON8_nWEzTxELBMWlfV7E6-6qi-g0DMEbVEz4,2885
 nucliadb/models/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/models/responses.py,sha256=qnuOoc7TrVSUnpikfTwHLKez47_DE4mSFzpxrwtqijA,1599
+nucliadb/models/internal/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
+nucliadb/models/internal/processing.py,sha256=bzPr-hXliY81zMUgG-PDyDiFKP7Xbs71s2d0SIAu4Do,4090
 nucliadb/purge/__init__.py,sha256=UXbto56EWYLwZj6uEc-flQVe3gDDNFtM6EV-aIkryPU,12353
 nucliadb/purge/orphan_shards.py,sha256=WSQmVQJQ-rAoQAmypOUJBpDP-9VJchWao3ZLTzHdJ1U,7764
 nucliadb/reader/__init__.py,sha256=C5Efic7WlGm2U2C5WOyquMFbIj2Pojwe_8mwzVYnOzE,1304
@@ -334,20 +336,20 @@ nucliadb/writer/api/constants.py,sha256=qWEDjFUycrEZnSJyLnNK4PQNodU2oVmkO4NycaEZ
 nucliadb/writer/api/utils.py,sha256=wIQHlU8RQiIGVLI72suvyVIKlCU44Unh0Ae0IiN6Qwo,1313
 nucliadb/writer/api/v1/__init__.py,sha256=akI9A_jloNLb0dU4T5zjfdyvmSAiDeIdjAlzNx74FlU,1128
 nucliadb/writer/api/v1/export_import.py,sha256=elf-EQY5DD3mhw8kWb9tQpDcbrF9sY6VFYqxQOjuVP0,8201
-nucliadb/writer/api/v1/field.py,sha256=FySCMpcruSAKGeepeAlOihjwxyUPcDO73Uilq5VDWRk,18514
+nucliadb/writer/api/v1/field.py,sha256=KOOBqBJzwsNczn_isxl-YFBL-bmduz3rzSDWMbAJefc,18523
 nucliadb/writer/api/v1/knowledgebox.py,sha256=PHEYDFa-sN5JrI8-EiVVg5FDOsRuCLT43kyAB4xt-xA,9530
 nucliadb/writer/api/v1/learning_config.py,sha256=CKBjqcbewkfPwGUPLDWzZSpro6XkmCaVppe5Qtpu5Go,3117
-nucliadb/writer/api/v1/resource.py,sha256=r5YTAZuHzaiBz1dLstWxvG74_LJGl2mgjpEsDmRB2qY,19777
+nucliadb/writer/api/v1/resource.py,sha256=jxphiyeXJq342BR1R8pRQ81L0i3Tczf_Yarqx_DqvWs,19786
 nucliadb/writer/api/v1/router.py,sha256=RjuoWLpZer6Kl2BW_wznpNo6XL3BOpdTGqXZCn3QrrQ,1034
 nucliadb/writer/api/v1/services.py,sha256=3AUjk-SmvqJx76v7y89DZx6oyasojPliGYeniRQjpcU,13337
 nucliadb/writer/api/v1/slug.py,sha256=xlVBDBpRi9bNulpBHZwhyftVvulfE0zFm1XZIWl-AKY,2389
 nucliadb/writer/api/v1/transaction.py,sha256=d2Vbgnkk_-FLGSTt3vfldwiJIUf0XoyD0wP1jQNz_DY,2430
-nucliadb/writer/api/v1/upload.py,sha256=hLMHXSaqEOE-vjKjhIupgdx8klJc3mVQp_oMwx5N-7o,33800
+nucliadb/writer/api/v1/upload.py,sha256=fwWXA5BuLPuGKhOcuyf0CdutWJITjJ6fAvDzV_X9VsU,33809
 nucliadb/writer/api/v1/vectorsets.py,sha256=F3iMViL5G95_Tns4aO2SOA0DwAzxK2_P8MXxtd_XLRE,6973
 nucliadb/writer/resource/__init__.py,sha256=cp15ZcFnHvpcu_5-aK2A4uUyvuZVV_MJn4bIXMa20ks,835
 nucliadb/writer/resource/audit.py,sha256=FvxMZPzrNHtd31HgpZEvxzwAkbxJTZRhPLqRYYJi3tA,1426
-nucliadb/writer/resource/basic.py,sha256=P2VXXXLKs43_Cd7Uvrcd-JTeuOJuUGu1Jpx8eujGi7Q,10451
-nucliadb/writer/resource/field.py,sha256=e5QGkR5ZDT1VUQgMXK7v6GGXJ2eek6jxGA0nPqjq_g4,20241
+nucliadb/writer/resource/basic.py,sha256=fjxZEsC_ftuRrpPDOQqSDfZR6JlVNSFPMckVGmjQ4lY,10426
+nucliadb/writer/resource/field.py,sha256=wL71wScbLVhu5LHv_aCUBFae6LhKtFr1aXKEubMSgcI,20366
 nucliadb/writer/resource/origin.py,sha256=pvhUDdU0mlWPUcpoQi4LDUJaRtfjzVVrA8XcGVI_N8k,2021
 nucliadb/writer/tus/__init__.py,sha256=huWpKnDnjsrKlBBJk30ta5vamlA-4x0TbPs_2Up8hyM,5443
 nucliadb/writer/tus/azure.py,sha256=XhWAlWTM0vmXcXtuEPYjjeEhuZjiZXZu8q9WsJ7omFE,4107
@@ -358,8 +360,8 @@ nucliadb/writer/tus/local.py,sha256=7jYa_w9b-N90jWgN2sQKkNcomqn6JMVBOVeDOVYJHto,
 nucliadb/writer/tus/s3.py,sha256=vF0NkFTXiXhXq3bCVXXVV-ED38ECVoUeeYViP8uMqcU,8357
 nucliadb/writer/tus/storage.py,sha256=ToqwjoYnjI4oIcwzkhha_MPxi-k4Jk3Lt55zRwaC1SM,2903
 nucliadb/writer/tus/utils.py,sha256=MSdVbRsRSZVdkaum69_0wku7X3p5wlZf4nr6E0GMKbw,2556
-nucliadb-6.3.5.post3990.dist-info/METADATA,sha256=nT9CHADQl_oE04SzbtUN2-ah8Rd2IHj9JnfasePEqy4,4301
-nucliadb-6.3.5.post3990.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-nucliadb-6.3.5.post3990.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
-nucliadb-6.3.5.post3990.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
-nucliadb-6.3.5.post3990.dist-info/RECORD,,
+nucliadb-6.3.5.post3996.dist-info/METADATA,sha256=4JGtXA-f0Q61lp2HVgY0ujHmW4VAQ0CP6MhdLtDlcNc,4301
+nucliadb-6.3.5.post3996.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+nucliadb-6.3.5.post3996.dist-info/entry_points.txt,sha256=XqGfgFDuY3zXQc8ewXM2TRVjTModIq851zOsgrmaXx4,1268
+nucliadb-6.3.5.post3996.dist-info/top_level.txt,sha256=hwYhTVnX7jkQ9gJCkVrbqEG1M4lT2F_iPQND1fCzF80,20
+nucliadb-6.3.5.post3996.dist-info/RECORD,,