nucliadb-models 6.9.6.post5453__py3-none-any.whl → 6.9.7.post5583__py3-none-any.whl

nucliadb_models/augment.py

@@ -13,45 +13,217 @@
 # limitations under the License.
 #
 
-from pydantic import BaseModel
+from enum import Enum
+from typing import Annotated
 
+from pydantic import BaseModel, Field, StringConstraints, model_validator
+from typing_extensions import Self
+
+from nucliadb_models import filters
 from nucliadb_models.common import FieldTypeName
 from nucliadb_models.resource import ExtractedDataTypeName, Resource
-from nucliadb_models.search import Image, ResourceProperties, SearchParamDefaults
+from nucliadb_models.search import Image, ResourceProperties
 
-ParagraphId = str
+ResourceIdPattern = r"^([0-9a-f]{32}|[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$"
+ResourceId = Annotated[
+    str,
+    StringConstraints(pattern=ResourceIdPattern, min_length=32, max_length=36),
+]
 
+FieldIdPattern = r"^[0-9a-f]{32}/[acftu]/[a-zA-Z0-9:_-]+(/[^/]{1,128})?$"
+FieldId = Annotated[
+    str,
+    StringConstraints(
+        pattern=FieldIdPattern,
+        min_length=32 + 1 + 1 + 1 + 1 + 0 + 0,
+        # max field id of 250
+        max_length=32 + 1 + 1 + 1 + 250 + 1 + 218,
+    ),
+]
 
-class AugmentedParagraph(BaseModel):
-    text: str | None = None
+ParagraphIdPattern = r"^[0-9a-f]{32}/[acftu]/[a-zA-Z0-9:_-]+(/[^/]{1,128})?/[0-9]+-[0-9]+$"
+ParagraphId = Annotated[
+    str,
+    StringConstraints(
+        # resource-uuid/field-type/field-id/[split-id/]paragraph-id
+        pattern=ParagraphIdPattern,
+        min_length=32 + 1 + 1 + 1 + 1 + 0 + 0 + 1 + 3,
+        # max field id of 250 and 10 digit paragraphs. More than enough
+        max_length=32 + 1 + 1 + 1 + 250 + 1 + 128 + 1 + 21,
+    ),
+]
 
-    neighbours_before: dict[ParagraphId, str] | None = None
-    neighbours_after: dict[ParagraphId, str] | None = None
 
-    image: Image | None = None
+# Request
 
 
-class AugmentedField(BaseModel):
-    page_preview_image: Image | None = None
+class ResourceProp(str, Enum):
+    """Superset of former `show` and `extracted` serializations options."""
 
+    # `show` props
+    BASIC = "basic"
+    ORIGIN = "origin"
+    EXTRA = "extra"
+    RELATIONS = "relations"
+    VALUES = "values"
+    ERRORS = "errors"
+    SECURITY = "security"
+    # `extracted` props
+    EXTRACTED_TEXT = "extracted_text"
+    EXTRACTED_METADATA = "extracted_metadata"
+    EXTRACTED_SHORTENED_METADATA = "extracted_shortened_metadata"
+    EXTRACTED_LARGE_METADATA = "extracted_large_metadata"
+    EXTRACTED_VECTOR = "extracted_vectors"
+    EXTRACTED_LINK = "extracted_link"
+    EXTRACTED_FILE = "extracted_file"
+    EXTRACTED_QA = "extracted_question_answers"
+    # new granular props
+    TITLE = "title"
+    SUMMARY = "summary"
+    CLASSIFICATION_LABELS = "classification_labels"
+
+    @classmethod
+    def from_show_and_extracted(
+        cls, show: list[ResourceProperties], extracted: list[ExtractedDataTypeName]
+    ) -> list["ResourceProp"]:
+        _show_to_prop = {
+            ResourceProperties.BASIC: cls.BASIC,
+            ResourceProperties.ORIGIN: cls.ORIGIN,
+            ResourceProperties.EXTRA: cls.EXTRA,
+            ResourceProperties.RELATIONS: cls.RELATIONS,
+            ResourceProperties.VALUES: cls.VALUES,
+            ResourceProperties.ERRORS: cls.ERRORS,
+            ResourceProperties.SECURITY: cls.SECURITY,
+        }
+        _extracted_to_prop = {
+            ExtractedDataTypeName.TEXT: cls.EXTRACTED_TEXT,
+            ExtractedDataTypeName.METADATA: cls.EXTRACTED_METADATA,
+            ExtractedDataTypeName.SHORTENED_METADATA: cls.EXTRACTED_SHORTENED_METADATA,
+            ExtractedDataTypeName.LARGE_METADATA: cls.EXTRACTED_LARGE_METADATA,
+            ExtractedDataTypeName.VECTOR: cls.EXTRACTED_VECTOR,
+            ExtractedDataTypeName.LINK: cls.EXTRACTED_LINK,
+            ExtractedDataTypeName.FILE: cls.EXTRACTED_FILE,
+            ExtractedDataTypeName.QA: cls.EXTRACTED_QA,
+        }
+
+        props = []
+        for s in show:
+            show_prop = _show_to_prop.get(s)
+            # show=extracted is not in the dict
+            if show_prop is None:
+                continue
+            props.append(show_prop)
+
+        if ResourceProperties.EXTRACTED in show:
+            for e in extracted:
+                extracted_prop = _extracted_to_prop[e]
+                props.append(extracted_prop)
+
+        return props
 
-class AugmentedResource(Resource):
-    def updated_from(self, origin: Resource):
-        for key in origin.model_fields.keys():
-            self.__setattr__(key, getattr(origin, key))
+
+class AugmentResourceFields(BaseModel):
+    text: bool = False
+    classification_labels: bool = False
+
+    filters: list[filters.Field | filters.Generated]
 
 
 class AugmentResources(BaseModel):
-    given: list[str]
+    given: list[ResourceId]
+
+    # TODO(decoupled-ask): replace this select for bool fields
+    select: list[ResourceProp] = Field(default_factory=list)
+
+    field_type_filter: list[FieldTypeName] | None = Field(
+        default=None,
+        deprecated="Only use this for legacy resource serialization",
+        title="Field type filter",
+        description=(
+            "Define which field types are serialized on resources of search results. "
+            "If omitted and legacy serialization is used, all field types will be serialized"
+        ),
+    )
 
-    show: list[ResourceProperties] = SearchParamDefaults.show.to_pydantic_field()
-    extracted: list[ExtractedDataTypeName] = SearchParamDefaults.extracted.to_pydantic_field()
-    field_type_filter: list[FieldTypeName] = SearchParamDefaults.field_type_filter.to_pydantic_field()
-    # TODO: field name filter, da field prefix filter
+    fields: AugmentResourceFields | None = None
+
+    @model_validator(mode="after")
+    def bwc_resource_serialization(self) -> Self:
+        if self.field_type_filter is not None and self.fields is not None:
+            raise ValueError("`field_type_filter` and `fields` are incompatible together")
+
+        return self
+
+
+class AugmentFields(BaseModel):
+    given: list[FieldId]
+
+    text: bool = False
+    classification_labels: bool = False
+    entities: bool = False  # also known as ners
+
+    # When enabled, augment all the messages from the conversation. This is
+    # incompatible with max_conversation_messages defined
+    full_conversation: bool = False
+
+    # When `full` disbled, this option controls the max amount of messages to be
+    # augmented. This number will be a best-effort window centered around the
+    # selected message. In addition, the 1st message of the conversation will
+    # always be included.
+    #
+    # This option is combinable with attachments.
+    max_conversation_messages: int | None = None
+
+    # Given a message, if it's a question, try to find an answer. Otherwise,
+    # return a window of messages following the requested one.
+    #
+    # This was previously done without explicit user consent, now it's an option.
+    conversation_answer_or_messages_after: bool = False
+
+    # Both attachment options will only add attachments for the full or the 1st
+    # + window, not answer nor messages after
+
+    # include conversation text attachments
+    conversation_text_attachments: bool = False
+    # include conversation image attachments
+    conversation_image_attachments: bool = False
+
+    @model_validator(mode="after")
+    def validate_cross_options(self):
+        if self.full_conversation and self.max_conversation_messages is not None:
+            raise ValueError(
+                "`full_conversation` and `max_conversation_messages` are not compatible together"
+            )
+        if (
+            (self.conversation_text_attachments or self.conversation_image_attachments)
+            and self.full_conversation is False
+            and self.max_conversation_messages is None
+        ):
+            raise ValueError(
+                "Attachments are only compatible with `full_conversation` and `max_conversation_messages`"
+            )
+        return self
+
+
+# TODO(decoupled-ask): remove unused metadata
+class ParagraphMetadata(BaseModel):
+    field_labels: list[str]
+    paragraph_labels: list[str]
+
+    is_an_image: bool
+    is_a_table: bool
+
+    # for extracted from visual content (ocr, inception, tables)
+    source_file: str | None
+
+    # for documents (pdf, docx...) only
+    page: int | None
+    in_page_with_visual: bool | None
 
 
 class AugmentParagraph(BaseModel):
     id: ParagraphId
+    metadata: ParagraphMetadata | None = None
 
 
 class AugmentParagraphs(BaseModel):
@@ -62,24 +234,106 @@ class AugmentParagraphs(BaseModel):
     neighbours_before: int = 0
     neighbours_after: int = 0
 
+    # TODO(decoupled-ask): implement image strategy
     # paragraph extracted from an image, return an image
     source_image: bool = False
 
+    # TODO(decoupled-ask): implement image strategy
     # paragraph extracted from a table, return table image
     table_image: bool = False
 
+    # TODO(decoupled-ask): implement image strategy
     # return page_preview instead of table image if table image enabled
     table_prefers_page_preview: bool = False
 
+    # TODO(decoupled-ask): implement image strategy
     # paragraph from a page, return page preview image
     page_preview_image: bool = False
 
 
 class AugmentRequest(BaseModel):
-    resources: AugmentResources
-    paragraphs: AugmentParagraphs
+    resources: AugmentResources | None = None
+    fields: AugmentFields | None = None
+    paragraphs: AugmentParagraphs | None = None
+
+
+# Response
+
+
+class AugmentedParagraph(BaseModel):
+    text: str | None = None
+
+    neighbours_before: list[ParagraphId] | None = None
+    neighbours_after: list[ParagraphId] | None = None
+
+    image: Image | None = None
+
+
+class AugmentedField(BaseModel):
+    text: str | None = None
+
+    classification_labels: dict[str, list[str]] | None = None
+
+    # former ners
+    entities: dict[str, list[str]] | None = None
+
+    page_preview_image: Image | None = None
+
+
+class AugmentedConversationMessage(BaseModel):
+    ident: str
+    text: str | None = None
+    attachments: list[FieldId] | None = None
+
+
+class AugmentedConversationField(BaseModel):
+    classification_labels: dict[str, list[str]] | None = None
+    # former ners
+    entities: dict[str, list[str]] | None = None
+
+    messages: list[AugmentedConversationMessage] | None = None
+
+    @property
+    def text(self) -> str | None:
+        """Syntactic sugar to access aggregate text from all messages"""
+        if self.messages is None:
+            return None
+
+        text = ""
+        for message in self.messages:
+            text += message.text or ""
+
+        return text or None
+
+    @property
+    def attachments(self) -> list[FieldId] | None:
+        """Syntactic sugar to access the aggregate of attachments from all messages."""
+        if self.messages is None:
+            return None
+
+        has_attachments = False
+        attachments = []
+        for message in self.messages:
+            if message.attachments is None:
+                continue
+            has_attachments = True
+            attachments.extend(message.attachments)
+
+        if has_attachments:
+            return attachments
+        else:
+            return None
+
+
+class AugmentedResource(Resource):
+    classification_labels: dict[str, list[str]] | None = None
+
+    def updated_from(self, origin: Resource):
+        for key in origin.model_fields.keys():
+            self.__setattr__(key, getattr(origin, key))
 
 
 class AugmentResponse(BaseModel):
-    resources: dict[str, AugmentedResource]
-    paragraphs: dict[str, AugmentedParagraph]
+    resources: dict[ResourceId, AugmentedResource]
+    fields: dict[FieldId, AugmentedField | AugmentedConversationField]
+    paragraphs: dict[ParagraphId, AugmentedParagraph]

nucliadb_models/common.py

@@ -108,7 +108,7 @@ class File(BaseModel):
         if self.md5 is None:
             # In case md5 is not supplied, compute it
             try:
-                result = hashlib.md5(base64.b64decode(self.payload))
+                result = hashlib.md5(base64.b64decode(self.payload), usedforsecurity=False)
                 self.md5 = result.hexdigest()
             except Exception:
                 raise ValueError("MD5 could not be computed")

nucliadb_models/filters.py

@@ -90,7 +90,7 @@ class Resource(FilterProp, extra="forbid"):
             try:
                 UUID(v)
             except ValueError:
-                raise ValueError("Invalid UUID")
+                raise ValueError(f"resource id filter '{v}' should be a valid UUID")
         return v
 
     @model_validator(mode="after")
@@ -276,7 +276,7 @@ class Generated(FilterProp, extra="forbid"):
     by: Literal["data-augmentation"] = pydantic.Field(
         description="Generator for this field. Currently, only data-augmentation is supported"
     )
-    da_task: Optional["str"] = pydantic.Field(
+    da_task: Optional[str] = pydantic.Field(
         default=None, description="Matches field generated by an specific DA task, given its prefix"
     )
 

nucliadb_models/resource.py

@@ -72,15 +72,6 @@ class ExtractedDataTypeName(str, Enum):
     QA = "question_answers"
 
 
-class ReleaseChannel(str, Enum):
-    """
-    Deprecated. No longer used.
-    """
-
-    STABLE = "STABLE"
-    EXPERIMENTAL = "EXPERIMENTAL"
-
-
 class KnowledgeBoxConfig(BaseModel):
     slug: Optional[SlugString] = Field(
         default=None, title="Slug", description="Slug for the Knowledge Box."

nucliadb_models/search.py

@@ -15,6 +15,7 @@
 import json
 from enum import Enum
 from typing import Any, Literal, Optional, Union
+from uuid import UUID
 
 from pydantic import BaseModel, Field, field_validator, model_validator
 from pydantic.aliases import AliasChoices
@@ -931,6 +932,13 @@ Please return ONLY the question without any explanation. Just the rephrased ques
                 values["top_k"] = SearchParamDefaults.top_k.default
         return values
 
+    @field_validator("resource_filters", mode="after")
+    def validate_resource_filters(cls, values: list[str]) -> list[str]:
+        if values is not None:
+            for v in values:
+                _validate_resource_filter(v)
+        return values
+
 
 class SearchRequest(BaseSearchRequest):
     features: list[SearchOptions] = SearchParamDefaults.search_features.to_pydantic_field(
@@ -1860,6 +1868,13 @@ Using this feature also disables the `citations` parameter. For maximal accuracy
             self.context = None
         return self
 
+    @field_validator("resource_filters", mode="after")
+    def validate_resource_filters(cls, values: list[str]) -> list[str]:
+        if values is not None:
+            for v in values:
+                _validate_resource_filter(v)
+        return values
+
 
 # Alias (for backwards compatiblity with testbed)
 class ChatRequest(AskRequest):
@@ -2062,8 +2077,10 @@ class FindResource(Resource):
     fields: dict[str, FindField]
 
     def updated_from(self, origin: Resource):
+        find_resource_model_fields = self.model_fields.keys()
         for key in origin.model_fields.keys():
-            self.__setattr__(key, getattr(origin, key))
+            if key in find_resource_model_fields:
+                self.__setattr__(key, getattr(origin, key))
 
 
 class KnowledgeboxFindResults(JsonBaseModel):
@@ -2524,3 +2541,22 @@ class CatalogFacetsRequest(BaseModel):
 
 class CatalogFacetsResponse(BaseModel):
     facets: dict[str, int]
+
+
+def _validate_resource_filter(v: str):
+    parts = v.split("/")
+
+    rid = parts[0]
+    try:
+        UUID(rid)
+    except ValueError:
+        raise ValueError(f"resource id filter '{rid}' should be a valid UUID")
+
+    if len(parts) > 1:
+        field_type = parts[1]
+        try:
+            FieldTypeName.from_abbreviation(field_type)
+        except KeyError:  # pragma: no cover
+            raise ValueError(
+                f"resource filter {v} has an invalid field type: {field_type}",
+            )

{nucliadb_models-6.9.6.post5453.dist-info → nucliadb_models-6.9.7.post5583.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: nucliadb_models
-Version: 6.9.6.post5453
+Version: 6.9.7.post5583
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://nuclia.com

{nucliadb_models-6.9.6.post5453.dist-info → nucliadb_models-6.9.7.post5583.dist-info}/RECORD

@@ -1,6 +1,6 @@
 nucliadb_models/__init__.py,sha256=3y8-htogKuCZcbhaUZdSjTeEjUSeec9aRWyL8AlKCyM,1077
-nucliadb_models/augment.py,sha256=vAtFh4D4eC4nvfwaRTlfeuAMOL9Z9TFZnUNiRAMasss,2543
-nucliadb_models/common.py,sha256=2dtKG4ZNi9p-yoNY76Uvyu1SlMeNYpH-MnuU3Q6w9Js,8169
+nucliadb_models/augment.py,sha256=b-A_qF3-MoIPTrK0ZjsGxV6mJA-TZqnHrSJT0vVnvqQ,10970
+nucliadb_models/common.py,sha256=gVG5kOhOwQZR-t5n3b9-hANMlLy2CHelUU5PPUf3bck,8192
 nucliadb_models/configuration.py,sha256=BBrJsNjP324Cw_5J3dBrGwvpkHQYbXEo3TUaI9IqAOg,2449
 nucliadb_models/content_types.py,sha256=36Ga-iGf4ivCqgtXC7imFgegrwHB117s9eqP62JtGv0,3456
 nucliadb_models/conversation.py,sha256=k9bKhkDiqhqmdrDfDPNoUfG7-2H_-KAyuOnETd8zV0E,5081
@@ -9,7 +9,7 @@ nucliadb_models/export_import.py,sha256=mNm9IArOLnC6TLupkwqVFhxD5d08mpIVOVFneECv
 nucliadb_models/external_index_providers.py,sha256=pL3leo4MkuJOnKlU1Sg6GT_mnK_VUBxGui-RPmDYVWU,1126
 nucliadb_models/extracted.py,sha256=Owz7LC3le3Dvau3TtRiO8NY84meOf6IxN-RrOqqpMPs,5593
 nucliadb_models/file.py,sha256=tXtgB9c7i2ADsnJ7HdbXyroAmXadGvOeA49htBh7BZo,2263
-nucliadb_models/filters.py,sha256=NQI2-4AFzzJuZy8NeY3jXlTbbU5wxiwMCP-5DrD-7lE,14759
+nucliadb_models/filters.py,sha256=MlyG3mWyOro90PLzzczVfyGrqRBD8RS1OMjVejm_tQ4,14793
 nucliadb_models/hydration.py,sha256=SlAzraJE6DX0uOpZWxu2k_9-ikYorsj0t8xwsWSBQZY,14363
 nucliadb_models/labels.py,sha256=9zqRgkpZuX3kUPwsTTgCH7JyOWK7dM5pwyuHJR86YdU,3949
 nucliadb_models/link.py,sha256=PF5hHLwdOed5TMBTxtokkgWtMh1bFnORZjybh0NwVCw,2526
@@ -17,8 +17,8 @@ nucliadb_models/metadata.py,sha256=OOKGy_83NtlG1QKQZEwMuwu4wbVEe7P30Y2QvnGSDto,8
 nucliadb_models/notifications.py,sha256=mna8-AoD_29Wds0Thl0AF0zpERnJmYGLZX1w1fUopMY,4036
 nucliadb_models/processing.py,sha256=nhKuHQjqCdb9zJVkYGPTLub23tK9e_lwL5OCDVymZjY,719
 nucliadb_models/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-nucliadb_models/resource.py,sha256=RzCos0QRgSMkaV-p7EoceSmt7UTzt9G9be5BKF-iGrQ,9021
-nucliadb_models/search.py,sha256=_vn3pDXcK4iwiCfim3BtlD5EaQAeXoxl2IfNDsrKesA,97514
+nucliadb_models/resource.py,sha256=k_8I_5cN4hrWEFfzpgxwKJmKfqTfnPfFH1drAXLUjKc,8881
+nucliadb_models/search.py,sha256=O7TA_U0ZUEdQbKAXxAQhmZHhv5kK8hU1wSrt14JCpPg,98669
 nucliadb_models/security.py,sha256=opxaDLfvk3aU0sjesK0jGrYLx5h4YCwlKKN0moYs_ig,1150
 nucliadb_models/synonyms.py,sha256=afbaVqSQSxGLwi2PusVaLSRpkOtA5AZmWOKd1f4nl2E,690
 nucliadb_models/text.py,sha256=60bxZnOjRHnDdezR8VfR3AZsXTOwePFPs2BKB8wxBak,3414
@@ -34,7 +34,7 @@ nucliadb_models/graph/responses.py,sha256=Sdq8OgFAL1YT-1lJyLLrkqcScvj7YTEqAUwQ-k
 nucliadb_models/internal/__init__.py,sha256=zG33bUz1rHFPtvqQPWn4rDwBJt3FJodGuQYD45quiQg,583
 nucliadb_models/internal/predict.py,sha256=Pnx6MmLfK65eExe1XnVxqmSlvMwdowewwks9BOEoqMw,2029
 nucliadb_models/internal/shards.py,sha256=__y1OZtWGiNcPQEWfSFOj8yw458WGi7mM4vZe0K-L1Y,1691
-nucliadb_models-6.9.6.post5453.dist-info/METADATA,sha256=x-3GFIapOwe9kpigwte5HkdfaNqPk-iL7CuE_NM3PAE,745
-nucliadb_models-6.9.6.post5453.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb_models-6.9.6.post5453.dist-info/top_level.txt,sha256=UrY1I8oeovIRwkXLYplssTrxQdUjhSEFDFbnwaIV3tA,16
-nucliadb_models-6.9.6.post5453.dist-info/RECORD,,
+nucliadb_models-6.9.7.post5583.dist-info/METADATA,sha256=cMtwMi17BpKGWqA1DpJCeDYb1Lr1gMSXcv3aVFZcrm8,745
+nucliadb_models-6.9.7.post5583.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb_models-6.9.7.post5583.dist-info/top_level.txt,sha256=UrY1I8oeovIRwkXLYplssTrxQdUjhSEFDFbnwaIV3tA,16
+nucliadb_models-6.9.7.post5583.dist-info/RECORD,,