nucliadb-models 6.8.1.post4983__py3-none-any.whl → 6.9.5.post5452__py3-none-any.whl

nucliadb_models/augment.py

@@ -0,0 +1,85 @@
+# Copyright 2025 Bosutech XXI S.L.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pydantic import BaseModel
+
+from nucliadb_models.common import FieldTypeName
+from nucliadb_models.resource import ExtractedDataTypeName, Resource
+from nucliadb_models.search import Image, ResourceProperties, SearchParamDefaults
+
+ParagraphId = str
+
+
+class AugmentedParagraph(BaseModel):
+    text: str | None = None
+
+    neighbours_before: dict[ParagraphId, str] | None = None
+    neighbours_after: dict[ParagraphId, str] | None = None
+
+    image: Image | None = None
+
+
+class AugmentedField(BaseModel):
+    page_preview_image: Image | None = None
+
+
+class AugmentedResource(Resource):
+    def updated_from(self, origin: Resource):
+        for key in origin.model_fields.keys():
+            self.__setattr__(key, getattr(origin, key))
+
+
+class AugmentResources(BaseModel):
+    given: list[str]
+
+    show: list[ResourceProperties] = SearchParamDefaults.show.to_pydantic_field()
+    extracted: list[ExtractedDataTypeName] = SearchParamDefaults.extracted.to_pydantic_field()
+    field_type_filter: list[FieldTypeName] = SearchParamDefaults.field_type_filter.to_pydantic_field()
+    # TODO: field name filter, da field prefix filter
+
+
+class AugmentParagraph(BaseModel):
+    id: ParagraphId
+
+
+class AugmentParagraphs(BaseModel):
+    given: list[AugmentParagraph]
+
+    text: bool = True
+
+    neighbours_before: int = 0
+    neighbours_after: int = 0
+
+    # paragraph extracted from an image, return an image
+    source_image: bool = False
+
+    # paragraph extracted from a table, return table image
+    table_image: bool = False
+
+    # return page_preview instead of table image if table image enabled
+    table_prefers_page_preview: bool = False
+
+    # paragraph from a page, return page preview image
+    page_preview_image: bool = False
+
+
+class AugmentRequest(BaseModel):
+    resources: AugmentResources
+    paragraphs: AugmentParagraphs
+
+
+class AugmentResponse(BaseModel):
+    resources: dict[str, AugmentedResource]
+    paragraphs: dict[str, AugmentedParagraph]

nucliadb_models/common.py

@@ -203,6 +203,15 @@ class FieldTypeName(str, Enum):
             "a": FieldTypeName.GENERIC,
         }[abbr]
 
+    def abbreviation(self) -> str:
+        return {
+            FieldTypeName.TEXT: "t",
+            FieldTypeName.FILE: "f",
+            FieldTypeName.LINK: "u",
+            FieldTypeName.CONVERSATION: "c",
+            FieldTypeName.GENERIC: "a",
+        }[self]
+
 
 class FieldRef(BaseModel):
     field_type: FieldTypeName

nucliadb_models/configuration.py

@@ -44,7 +44,7 @@ def _model_fields(model: type[BaseModel], skip: list[str]) -> dict[str, Any]:
     }
 
 
-# FindConfig is a FindConfig without `search_configuration`
+# FindConfig is a FindRequest without `search_configuration`
 FindConfig = create_model("FindConfig", **_model_fields(FindRequest, skip=["search_configuration"]))
 
 

nucliadb_models/conversation.py

@@ -86,10 +86,10 @@ class FieldConversation(BaseModel):
 
 
 class InputMessageContent(BaseModel):
-    text: str
+    text: str = Field()
     format: MessageFormat = MessageFormat.PLAIN
-    attachments: List[FileB64] = []
-    attachments_fields: List[FieldRef] = []
+    attachments: List[FileB64] = Field(default=[], max_length=50)
+    attachments_fields: List[FieldRef] = Field(default=[], max_length=50)
 
 
 class InputMessage(BaseModel):
@@ -102,10 +102,12 @@ class InputMessage(BaseModel):
     to: List[str] = Field(
         default_factory=list,
         description="List of recipients of the message, e.g. ['assistant'] or ['user']",
+        max_length=100,
     )
     content: InputMessageContent
     ident: str = Field(
-        description="Unique identifier for the message. Must be unique within the conversation."
+        description="Unique identifier for the message. Must be unique within the conversation.",
+        max_length=128,
     )
     type_: Optional[MessageType] = Field(None, alias="type")
 
@@ -126,7 +128,7 @@ class InputMessage(BaseModel):
 class InputConversationField(BaseModel):
     messages: List[InputMessage] = Field(
         default_factory=list,
-        description="List of messages in the conversation field. Each message must have a unique ident.",
+        description="List of messages in the conversation field. Each message must have a unique ident. A single conversation can contain up to 51,200 messages. You can add up to 2,048 messages per request.",
     )
     extract_strategy: Optional[str] = Field(
         default=None,

nucliadb_models/external_index_providers.py

@@ -22,32 +22,18 @@ from pydantic import BaseModel
 class ExternalIndexProviderType(str, Enum):
     """
     Enum for the different external index providers.
-    For now only Pinecone is supported, but we may add more in the future.
+    For now none are supported, but we may add some in the future.
     """
 
-    PINECONE = "pinecone"
+    UNSET = "unset"
 
 
 class ExternalIndexProviderBase(BaseModel):
     type: ExternalIndexProviderType
 
 
-class PineconeServerlessCloud(str, Enum):
-    """
-    List of cloud providers supported by Pinecone serverless vector database.
-    """
-
-    AWS_US_EAST_1 = "aws_us_east_1"
-    AWS_US_WEST_2 = "aws_us_west_2"
-    AWS_EU_WEST_1 = "aws_eu_west_1"
-    GCP_US_CENTRAL1 = "gcp_us_central1"
-    AZURE_EASTUS2 = "azure_eastus2"
-
-
-class PineconeIndexProvider(ExternalIndexProviderBase):
-    type: ExternalIndexProviderType = ExternalIndexProviderType.PINECONE
-    api_key: str
-    serverless_cloud: PineconeServerlessCloud
+class DummyIndexProvider(ExternalIndexProviderBase):
+    type: ExternalIndexProviderType = ExternalIndexProviderType.UNSET
 
 
-ExternalIndexProvider = Union[PineconeIndexProvider,]
+ExternalIndexProvider = Union[DummyIndexProvider,]

nucliadb_models/hydration.py

@@ -0,0 +1,425 @@
+# Copyright 2025 Bosutech XXI S.L.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from typing import Annotated, Optional, Union
+
+from pydantic import BaseModel, Field, StringConstraints
+
+from nucliadb_models.common import FieldTypeName
+from nucliadb_models.metadata import Origin
+from nucliadb_models.resource import FieldConversation, FieldFile, FieldLink, FieldText
+from nucliadb_models.search import Image
+from nucliadb_models.security import ResourceSecurity
+
+
+class ResourceHydration(BaseModel, extra="forbid"):
+    title: bool = Field(
+        default=True,
+        description="Hydrate resource titles",
+    )
+    summary: bool = Field(
+        default=False,
+        description="Hydrate resource summaries",
+    )
+
+    origin: bool = Field(
+        default=False,
+        description="Hydrate resource origin",
+    )
+
+    security: bool = Field(
+        default=False,
+        description="Hydrate resource security metadata",
+    )
+
+
+class TextFieldHydration(BaseModel, extra="forbid"):
+    value: bool = Field(
+        default=False,
+        description="Hydrate text field values. Field values are similar payloads to the ones used to create them",
+    )
+    extracted_text: bool = Field(
+        default=False,
+        description="Hydrate extracted text for text fields",
+    )
+    # TODO: what else should be interesting to add?
+
+
+class FileFieldHydration(BaseModel, extra="forbid"):
+    value: bool = Field(
+        default=False,
+        description="Hydrate file field values. Field values are similar payloads to the ones used to create them",
+    )
+    extracted_text: bool = Field(
+        default=False,
+        description="Hydrate extracted text for file fields",
+    )
+    # TODO: what else should be interesting to add?
+
+
+class LinkFieldHydration(BaseModel, extra="forbid"):
+    value: bool = Field(
+        default=False,
+        description="Hydrate link field values. Field values are similar payloads to the ones used to create them",
+    )
+    extracted_text: bool = Field(
+        default=False,
+        description="Hydrate extracted text for link fields",
+    )
+    # TODO: what else should be interesting to add?
+
+
+class ConversationFieldHydration(BaseModel, extra="forbid"):
+    value: bool = Field(
+        default=False,
+        description="Hydrate conversation field values. Field values are similar payloads to the ones used to create them",
+    )
+
+    # TODO: add fields to hydrate conversation fields. Think about how to handle
+    # splits and fulfill the conversational RAG strategies
+
+    # TODO: what else should be interesting to add?
+
+
+class GenericFieldHydration(BaseModel, extra="forbid"):
+    value: bool = Field(
+        default=False,
+        description="Hydrate generic field values. Field values are similar payloads to the ones used to create them",
+    )
+    extracted_text: bool = Field(
+        default=False,
+        description="Hydrate extracted text for generic fields",
+    )
+    # TODO: what else should be interesting to add?
+
+
+class FieldHydration(BaseModel, extra="forbid"):
+    text: Optional[TextFieldHydration] = Field(
+        default_factory=TextFieldHydration,
+        description="Text fields hydration options",
+    )
+    file: Optional[FileFieldHydration] = Field(
+        default_factory=FileFieldHydration,
+        description="File fields hydration options",
+    )
+    link: Optional[LinkFieldHydration] = Field(
+        default_factory=LinkFieldHydration,
+        description="Link fields hydration options",
+    )
+    conversation: Optional[ConversationFieldHydration] = Field(
+        default_factory=ConversationFieldHydration,
+        description="Conversation fields hydration options",
+    )
+    generic: Optional[GenericFieldHydration] = Field(
+        default_factory=GenericFieldHydration,
+        description="Generic fields hydration options",
+    )
+
+
+class NeighbourParagraphHydration(BaseModel, extra="forbid"):
+    before: int = Field(
+        default=2,
+        ge=0,
+        description="Number of previous paragraphs to hydrate",
+    )
+    after: int = Field(
+        default=2,
+        ge=0,
+        description="Number of following paragraphs to hydrate",
+    )
+
+
+class RelatedParagraphHydration(BaseModel, extra="forbid"):
+    neighbours: Optional[NeighbourParagraphHydration] = Field(
+        default=None,
+        description="Hydrate extra paragraphs that surround the original one",
+    )
+
+    # TODO: FEATURE: implement related paragraphs by page
+    # page: bool = Field(
+    #     default=False,
+    #     description="Hydrate all paragraphs in the same page. This only applies to fields with pages",
+    # )
+
+    # TODO: description
+    # XXX: should we let users control the amount of elements?
+    parents: bool = False
+    # TODO: description
+    # XXX: should we let users control the amount of elements?
+    siblings: bool = False
+    # TODO: description
+    # XXX: should we let users control the amount of elements?
+    replacements: bool = False
+
+
+class ImageParagraphHydration(BaseModel, extra="forbid"):
+    # The source image is also known as reference or reference_file in the
+    # paragraph context. The reference/reference_file is the filename of the
+    # source image from which the paragraph has been extracted
+    source_image: bool = Field(
+        default=False,
+        description=(
+            "When a paragraph has been extracted from an image (using OCR, inception...), "
+            "hydrate the image that represents it"
+        ),
+    )
+
+
+class TableParagraphHydration(BaseModel, extra="forbid"):
+    # TODO: implement. ARAG uses the label "/k/table" to check whether a
+    # paragraph is or a table or not. We can also use info on maindb
+    table_page_preview: bool = Field(
+        default=False,
+        description="Hydrate the page preview for the table. This will only hydrate fields with pages",
+    )
+
+
+class ParagraphPageHydration(BaseModel, extra="forbid"):
+    # For some field types (file and link) learning generates previews. A
+    # preview is a PDF file representing the content. For a docx for example, is
+    # the PDF equivalent. Depending on the field type, the preview can
+    # represent, for example, a page in a document or a portion of a webpage.
+    page_with_visual: bool = Field(
+        default=False,
+        description=(
+            "When a paragraph has been extracted from a page containing visual "
+            "content (images, tables...), hydrate the preview of the paragraph's "
+            "page as an image. Not all field types have previews nor visual content"
+        ),
+    )
+
+
+class ParagraphHydration(BaseModel, extra="forbid"):
+    text: bool = Field(
+        default=True,
+        description="Hydrate paragraph text",
+    )
+    image: Optional[ImageParagraphHydration] = Field(
+        default=None,
+        description="Hydrate options for paragraphs extracted from images (using OCR, inception...)",
+    )
+    table: Optional[TableParagraphHydration] = Field(
+        default=None,
+        description="Hydrate options for paragraphs extracted from tables",
+    )
+
+    # TODO: at some point, we should add hydration options for paragraphs from
+    # audio and video
+
+    page: Optional[ParagraphPageHydration] = Field(
+        default=None,
+        description="Hydrte options for paragraphs within a page. This applies to paragraphs in fields with pages",
+    )
+
+    related: Optional[RelatedParagraphHydration] = Field(
+        default=None,
+        description="Hydration options for related paragraphs. For example, neighbours or sibling paragraphs",
+    )
+
+
+class Hydration(BaseModel, extra="forbid"):
+    resource: Optional[ResourceHydration] = Field(
+        default_factory=ResourceHydration,
+        description="Resource hydration options",
+    )
+    field: FieldHydration = Field(
+        default_factory=FieldHydration,
+        description="Field hydration options",
+    )
+    paragraph: ParagraphHydration = Field(
+        default_factory=ParagraphHydration,
+        description="Paragraph hydration options",
+    )
+
+
+ParagraphId = Annotated[
+    str,
+    StringConstraints(
+        # resource-uuid/field-type/field-id/[split-id/]paragraph-id
+        pattern=r"^[0-9a-f]{32}/[acftu]/[a-zA-Z0-9:_-]+(/[^/]{1,128})?/[0-9]+-[0-9]+$",
+        min_length=32 + 1 + 1 + 1 + 1 + 0 + 0 + 1 + 3,
+        # max field id of 250 and 10 digit paragraphs. More than enough
+        max_length=32 + 1 + 1 + 1 + 250 + 1 + 128 + 1 + 21,
+    ),
+]
+
+
+class HydrateRequest(BaseModel, extra="forbid"):
+    data: list[ParagraphId] = Field(
+        description="List of paragraph ids we want to hydrate",
+        max_length=50,
+    )
+    hydration: Hydration = Field(description="Description of how hydration must be performed")
+
+
+### Response models
+
+
+class HydratedResource(BaseModel, extra="forbid"):
+    id: str = Field(description="Unique resource id")
+    slug: str = Field(description="Resource slug")
+
+    title: Optional[str] = None
+    summary: Optional[str] = None
+
+    origin: Optional[Origin] = None
+
+    security: Optional[ResourceSecurity] = None
+
+    # TODO: add resource labels to hydrated resources
+
+
+class FieldExtractedData(BaseModel, extra="forbid"):
+    text: Optional[str] = None
+
+
+class SplitFieldExtractedData(BaseModel, extra="forbid"):
+    texts: Optional[dict[str, str]] = None
+
+
+class HydratedTextField(BaseModel, extra="forbid"):
+    id: str = Field("Unique field id")
+    resource: str = Field("Field resource id")
+    field_type: FieldTypeName = FieldTypeName.TEXT
+
+    value: Optional[FieldText] = None
+    extracted: Optional[FieldExtractedData] = None
+
+
+class HydratedFileField(BaseModel, extra="forbid"):
+    id: str = Field("Unique field id")
+    resource: str = Field("Field resource id")
+    field_type: FieldTypeName = FieldTypeName.FILE
+
+    value: Optional[FieldFile] = None
+    extracted: Optional[FieldExtractedData] = None
+
+    previews: Optional[dict[str, Image]] = Field(
+        default=None,
+        title="Previews of specific parts of the field",
+        description=(
+            "Previews for specific pages of this field. Previews are differents"
+            "depending on the file type. For example, for a PDF file, a preview"
+            "will be an image of a single page."
+            "In this field, previews will be populated according to the hydration"
+            "options requested."
+        ),
+    )
+
+
+class HydratedLinkField(BaseModel, extra="forbid"):
+    id: str = Field("Unique field id")
+    resource: str = Field("Field resource id")
+    field_type: FieldTypeName = FieldTypeName.LINK
+
+    value: Optional[FieldLink] = None
+    extracted: Optional[FieldExtractedData] = None
+
+
+class HydratedConversationField(BaseModel, extra="forbid"):
+    id: str = Field("Unique field id")
+    resource: str = Field("Field resource id")
+    field_type: FieldTypeName = FieldTypeName.CONVERSATION
+
+    value: Optional[FieldConversation] = None
+    extracted: Optional[FieldExtractedData] = None
+
+
+class HydratedGenericField(BaseModel, extra="forbid"):
+    id: str = Field("Unique field id")
+    resource: str = Field("Field resource id")
+    field_type: FieldTypeName = FieldTypeName.TEXT
+
+    value: Optional[str] = None
+    extracted: Optional[FieldExtractedData] = None
+
+
+class RelatedNeighbourParagraphRefs(BaseModel, extra="forbid"):
+    before: Optional[list[str]] = None
+    after: Optional[list[str]] = None
+
+
+class RelatedParagraphRefs(BaseModel, extra="forbid"):
+    neighbours: Optional[RelatedNeighbourParagraphRefs] = None
+    parents: Optional[list[str]] = None
+    siblings: Optional[list[str]] = None
+    replacements: Optional[list[str]] = None
+
+
+class HydratedParagraphImage(BaseModel, extra="forbid"):
+    source_image: Optional[Image] = Field(
+        default=None,
+        description=(
+            "Source image for this paragraph. This only applies to paragraphs "
+            "extracted from an image using OCR or inception, and if this "
+            "hydration option has been enabled in the request"
+        ),
+    )
+
+
+class HydratedParagraphTable(BaseModel, extra="forbid"):
+    page_preview_ref: Optional[str] = Field(
+        default=None,
+        description=(
+            "Referento to the page preview for this paragraph. The actual "
+            "preview will be found in the previews of its field. This only "
+            "applies to paragraphs generated from a table and if the "
+            "corresponding hydration option has been enabled in the request"
+        ),
+    )
+
+
+class HydratedParagraphPage(BaseModel, extra="forbid"):
+    page_preview_ref: Optional[str] = Field(
+        default=None,
+        description=(
+            "Reference to the page preview for this paragraph. The actual "
+            "preview will be found in the previews of its field. This only "
+            "applies to paragraphs extracted from a page containing visual "
+            "content and if the corresponding hydration option has been enabled "
+            "in the request"
+        ),
+    )
+
+
+class HydratedParagraph(BaseModel, extra="forbid"):
+    id: str = Field(description="Unique paragraph id")
+    field: str = Field(description="Paragraph field id")
+    resource: str = Field(description="Paragraph resource id")
+
+    text: Optional[str] = None
+
+    # TODO: add labels to hydrated paragraphs
+    # labels: Optional[list[str]] = None
+
+    related: Optional[RelatedParagraphRefs] = None
+
+    image: Optional[HydratedParagraphImage] = None
+    table: Optional[HydratedParagraphTable] = None
+    page: Optional[HydratedParagraphPage] = None
+
+
+class Hydrated(BaseModel, extra="forbid"):
+    resources: dict[str, HydratedResource]
+    fields: dict[
+        str,
+        Union[
+            HydratedTextField,
+            HydratedFileField,
+            HydratedLinkField,
+            HydratedConversationField,
+            HydratedGenericField,
+        ],
+    ]
+    paragraphs: dict[str, HydratedParagraph]

nucliadb_models/metadata.py

@@ -17,7 +17,7 @@ from datetime import datetime
 from enum import Enum
 from typing import Any, Dict, List, Optional
 
-from pydantic import BaseModel, Field, model_validator
+from pydantic import BaseModel, Field, field_validator, model_validator
 from typing_extensions import Self
 
 from nucliadb_models.utils import DateTime
@@ -231,15 +231,24 @@ class InputOrigin(BaseModel):
         default=[],
         title="Tags",
         description="Resource tags about the origin system. It can later be used for filtering on search endpoints with '/origin.tags/{tag}'",
+        max_length=300,
     )
-    collaborators: List[str] = []
+    collaborators: List[str] = Field(default=[], max_length=100)
     filename: Optional[str] = None
-    related: List[str] = []
+    related: List[str] = Field(default=[], max_length=100)
     path: Optional[str] = Field(
         default=None,
         description="Path of the original resource. Typically used to store folder structure information of the resource at the origin system. It can be later used for filtering on search endpoints with '/origin.path/{path}'",
+        max_length=2048,
     )
 
+    @field_validator("tags")
+    def validate_tag_length(cls, tags):
+        for tag in tags:
+            if len(tag) > 512:
+                raise ValueError("Each tag must be at most 1024 characters long")
+        return tags
+
 
 class Origin(InputOrigin):
     # Created and modified are redefined to
@@ -247,6 +256,12 @@ class Origin(InputOrigin):
     created: Optional[datetime] = None
     modified: Optional[datetime] = None
 
+    tags: List[str] = Field(
+        default=[],
+        title="Tags",
+        description="Resource tags about the origin system. It can later be used for filtering on search endpoints with '/origin.tags/{tag}'",
+    )
+
     class Source(Enum):
         WEB = "WEB"
         DESKTOP = "DESKTOP"

nucliadb_models/search.py

@@ -79,8 +79,9 @@ ANSWER_JSON_SCHEMA_EXAMPLE = {
 class ModelParamDefaults:
     applied_autofilters = ParamDefault(
         default=[],
-        title="Autofilters",
-        description="List of filters automatically applied to the search query",
+        title="Applied autofilters",
+        description="[deprecated] list of filters automatically applied to the search query",
+        deprecated=True,
     )
 
 
@@ -296,6 +297,8 @@ class KnowledgeboxSearchResults(JsonBaseModel):
     relations: Optional[Relations] = None
     nodes: Optional[list[dict[str, str]]] = None
     shards: Optional[list[str]] = None
+
+    # TODO: remove on a future major release
     autofilters: list[str] = ModelParamDefaults.applied_autofilters.to_pydantic_field()
 
 
@@ -344,10 +347,12 @@ SortOrderMap = {
 
 class SortOptions(BaseModel):
     field: SortField
-    limit: Optional[int] = Field(None, gt=0)
     order: SortOrder = SortOrder.DESC
 
 
+MAX_RANK_FUSION_WINDOW = 500
+
+
 class RankFusionName(str, Enum):
     RECIPROCAL_RANK_FUSION = "rrf"
 
@@ -377,7 +382,7 @@ class ReciprocalRankFusion(_BaseRankFusion):
     )
     window: Optional[int] = Field(
         default=None,
-        le=500,
+        le=MAX_RANK_FUSION_WINDOW,
         title="RRF window",
         description="Number of elements for retrieval to do RRF. Window must be greater or equal to top_k. Greater values will increase probability of multi match at cost of retrieval time",  # noqa: E501
     )
@@ -481,11 +486,6 @@ class SearchParamDefaults:
         description="The list of facets to calculate. The facets follow the same syntax as filters: https://docs.nuclia.dev/docs/rag/advanced/search-filters",  # noqa: E501
         max_items=50,
     )
-    autofilter = ParamDefault(
-        default=False,
-        title="Automatic search filtering",
-        description="If set to true, the search will automatically add filters to the query. For example, it will filter results containing the entities detected in the query",  # noqa: E501
-    )
     chat_query = ParamDefault(
         default=...,
         title="Query",
@@ -505,10 +505,18 @@ class SearchParamDefaults:
     )
     top_k = ParamDefault(
         default=20,
+        gt=-1,
         le=200,
         title="Top k",
         description="The number of results search should return. The maximum number of results allowed is 200.",
     )
+    offset = ParamDefault(
+        default=0,
+        gt=-1,
+        le=1000,
+        title="Results offset",
+        description="The number of results to skip, starting from the beginning in sort order. Used for pagination. It can only be used with the keyword and fulltext indexes.",
+    )
     highlight = ParamDefault(
         default=False,
         title="Highlight",
@@ -534,12 +542,6 @@ class SearchParamDefaults:
         title="Sort order",
         description="Order to sort results with",
     )
-    sort_limit = ParamDefault(
-        default=None,
-        title="Sort limit",
-        description="",
-        gt=0,
-    )
     sort_field = ParamDefault(
         default=None,
         title="Sort field",
@@ -876,7 +878,9 @@ class BaseSearchRequest(AuditMetadataBase):
     vectorset: Optional[str] = SearchParamDefaults.vectorset.to_pydantic_field()
     with_duplicates: bool = SearchParamDefaults.with_duplicates.to_pydantic_field()
     with_synonyms: bool = SearchParamDefaults.with_synonyms.to_pydantic_field()
-    autofilter: bool = SearchParamDefaults.autofilter.to_pydantic_field()
+    # autofilter is deprecated and its logic was removed. We're just keeping it in the model definition to
+    # avoid breaking changes in the python sdks. Please remove on a future major release.
+    autofilter: SkipJsonSchema[bool] = False
     resource_filters: list[str] = SearchParamDefaults.resource_filters.to_pydantic_field()
     security: Optional[RequestSecurity] = SearchParamDefaults.security.to_pydantic_field()
     show_hidden: bool = SearchParamDefaults.show_hidden.to_pydantic_field()
@@ -938,12 +942,32 @@ class SearchRequest(BaseSearchRequest):
     )
     faceted: list[str] = SearchParamDefaults.faceted.to_pydantic_field()
     sort: Optional[SortOptions] = SearchParamDefaults.sort.to_pydantic_field()
+    offset: int = SearchParamDefaults.offset.to_pydantic_field()
 
     @field_validator("faceted")
     @classmethod
     def nested_facets_not_supported(cls, facets):
         return validate_facets(facets)
 
+    @model_validator(mode="after")
+    def offset_sort_only_on_keyword_indexes(self):
+        has_non_keyword_indexes = set(self.features) & {SearchOptions.SEMANTIC, SearchOptions.RELATIONS}
+        if has_non_keyword_indexes:
+            if self.offset > 0:
+                raise ValueError("offset cannot be used with the semantic or relations index")
+            if self.sort and self.sort.field != SortField.SCORE:
+                raise ValueError("sort by date cannot be used with the semantic or relations index")
+
+        return self
+
+    @field_validator("sort", mode="after")
+    @classmethod
+    def sorting_by_title_not_supported(cls, value: Optional[SortOptions]) -> Optional[SortOptions]:
+        if value and value.field == SortField.TITLE:
+            raise ValueError("sorting by title not supported in /search")
+
+        return value
+
 
 class Author(str, Enum):
     NUCLIA = "NUCLIA"
@@ -1008,6 +1032,12 @@ class Reasoning(BaseModel):
     )
 
 
+class CitationsType(str, Enum):
+    NONE = "none"
+    DEFAULT = "default"
+    LLM_FOOTNOTES = "llm_footnotes"
+
+
 class ChatModel(BaseModel):
     """
     This is the model for the predict request payload on the chat endpoint
@@ -1039,10 +1069,16 @@ class ChatModel(BaseModel):
     user_prompt: Optional[UserPrompt] = Field(
         default=None, description="Optional custom prompt input by the user"
     )
-    citations: bool = Field(default=False, description="Whether to include the citations in the answer")
+    citations: Union[bool, None, CitationsType] = Field(
+        default=None,
+        description="Whether to include citations in the response. "
+        "If set to None or False, no citations will be computed. "
+        "If set to True or 'default', citations will be computed after answer generation and send as a separate `CitationsGenerativeResponse` chunk. "
+        "If set to 'llm_footnotes', citations will be included in the LLM's response as markdown-styled footnotes. A `FootnoteCitationsGenerativeResponse` chunk will also be sent to map footnote ids to context keys in the `query_context`.",
+    )
     citation_threshold: Optional[float] = Field(
         default=None,
-        description="If citations is True, this sets the similarity threshold (0 to 1) for paragraphs to be included as citations. Lower values result in more citations. If not provided, Nuclia's default threshold is used.",  # noqa: E501
+        description="If citations is set to True or 'default', this will be the similarity threshold. Value between 0 and 1, lower values will produce more citations. If not set, it will be set to the optimized threshold found by Nuclia.",
         ge=0.0,
         le=1.0,
     )
@@ -1158,7 +1194,7 @@ ALLOWED_FIELD_TYPES: dict[str, str] = {
     "t": "text",
     "f": "file",
     "u": "link",
-    "d": "datetime",
+    "c": "conversation",
     "a": "generic",
 }
 
@@ -1166,16 +1202,19 @@ ALLOWED_FIELD_TYPES: dict[str, str] = {
 class FieldExtensionStrategy(RagStrategy):
     name: Literal["field_extension"] = "field_extension"
     fields: list[str] = Field(
+        default=[],
         title="Fields",
-        description="List of field ids to extend the context with. It will try to extend the retrieval context with the specified fields in the matching resources. The field ids have to be in the format `{field_type}/{field_name}`, like 'a/title', 'a/summary' for title and summary fields or 't/amend' for a text field named 'amend'.",  # noqa: E501
-        min_length=1,
+        description="List of field ids to extend the context with. It will try to extend the retrieval context with the specified fields in the matching resources. The field ids have to be in the format `{field_type}/{field_name}`, like 'a/title', 'a/summary' for title and summary fields or 't/amend' for a text field named 'amend'.",
+    )
+    data_augmentation_field_prefixes: list[str] = Field(
+        default=[],
+        description="List of prefixes for data augmentation added fields to extend the context with. For example, if the prefix is 'simpson', all fields that are a result of data augmentation with that prefix will be used to extend the context.",
     )
 
-    @field_validator("fields", mode="after")
-    @classmethod
-    def fields_validator(cls, fields) -> Self:
+    @model_validator(mode="after")
+    def field_extension_strategy_validator(self) -> Self:
         # Check that the fields are in the format {field_type}/{field_name}
-        for field in fields:
+        for field in self.fields:
             try:
                 field_type, _ = field.strip("/").split("/")
             except ValueError:
@@ -1188,8 +1227,7 @@ class FieldExtensionStrategy(RagStrategy):
                     f"Field '{field}' does not have a valid field type. "
                     f"Valid field types are: {allowed_field_types_part}."
                 )
-
-        return fields
+        return self
 
 
 class FullResourceApplyTo(BaseModel):
@@ -1227,6 +1265,7 @@ class HierarchyResourceStrategy(RagStrategy):
         title="Count",
         description="Number of extra characters that are added to each matching paragraph when adding to the context.",
         ge=0,
+        le=1024,
     )
 
 
@@ -1455,7 +1494,7 @@ class PageImageStrategy(ImageRagStrategy):
     count: Optional[int] = Field(
         default=None,
         title="Count",
-        description="Maximum number of images to retrieve from the page. By default, at most 5 images are retrieved.",
+        description="Maximum number of page images to retrieve. By default, at most 5 images are retrieved.",
     )
 
 
@@ -1606,7 +1645,11 @@ class AskRequest(AuditMetadataBase):
         description="Image that will be used together with the query text for retrieval and then sent to the LLM as part of the context. "
         "If a query image is provided, the `extra_context_images` and `rag_images_strategies` will be disabled.",
     )
-    autofilter: bool = SearchParamDefaults.autofilter.to_pydantic_field()
+
+    # autofilter is deprecated and its logic was removed. We're just keeping it in the model definition to
+    # avoid breaking changes in the python sdks. Please remove on a future major release.
+    autofilter: SkipJsonSchema[bool] = False
+
     highlight: bool = SearchParamDefaults.highlight.to_pydantic_field()
     resource_filters: list[str] = SearchParamDefaults.resource_filters.to_pydantic_field()
     prompt: Optional[Union[str, CustomPrompt]] = Field(
@@ -1616,13 +1659,16 @@ class AskRequest(AuditMetadataBase):
     )
     rank_fusion: Union[RankFusionName, RankFusion] = SearchParamDefaults.rank_fusion.to_pydantic_field()
     reranker: Union[RerankerName, Reranker] = SearchParamDefaults.reranker.to_pydantic_field()
-    citations: bool = Field(
-        default=False,
-        description="Whether to include the citations for the answer in the response",
+    citations: Union[bool, None, CitationsType] = Field(
+        default=None,
+        description="Whether to include citations in the response. "
+        "If set to None or False, no citations will be computed. "
+        "If set to True or 'default', citations will be computed after answer generation and send as a separate `CitationsGenerativeResponse` chunk. "
+        "If set to 'llm_footnotes', citations will be included in the LLM's response as markdown-styled footnotes. A `FootnoteCitationsGenerativeResponse` chunk will also be sent to map footnote ids to context keys in the `query_context`.",
     )
     citation_threshold: Optional[float] = Field(
         default=None,
-        description="If citations is True, this sets the similarity threshold (0 to 1) for paragraphs to be included as citations. Lower values result in more citations. If not provided, Nuclia's default threshold is used.",
+        description="If citations is set to True or 'default', this will be the similarity threshold. Value between 0 and 1, lower values will produce more citations. If not set, it will be set to the optimized threshold found by Nuclia.",
         ge=0.0,
         le=1.0,
     )
@@ -2257,10 +2303,15 @@ class SyncAskResponse(BaseModel):
         description="The detected relations of the answer",
     )
     citations: dict[str, Any] = Field(
-        default={},
+        default_factory=dict,
         title="Citations",
         description="The citations of the answer. List of references to the resources used to generate the answer.",
     )
+    citation_footnote_to_context: dict[str, str] = Field(
+        default_factory=dict,
+        title="Citation footnote to context",
+        description="""Maps ids in the footnote citations to query_context keys (normally paragraph ids)""",
+    )
     augmented_context: Optional[AugmentedContext] = Field(
         default=None,
         description=(
@@ -2370,6 +2421,18 @@ class CitationsAskResponseItem(BaseModel):
     citations: dict[str, Any]
 
 
+class FootnoteCitationsAskResponseItem(BaseModel):
+    type: Literal["footnote_citations"] = "footnote_citations"
+    footnote_to_context: dict[str, str] = Field(
+        description="""Maps ids in the footnote citations to query_context keys (normally paragraph ids)
+e.g.,
+{ "block-AA": "f44f4e8acbfb1d48de3fd3c2fb04a885/f/f44f4e8acbfb1d48de3fd3c2fb04a885/73758-73972", ... }
+If the query_context is a list, it will map to 1-based indices as strings
+e.g., { "block-AA": "1", "block-AB": "2", ... }
+"""
+    )
+
+
 class StatusAskResponseItem(BaseModel):
     type: Literal["status"] = "status"
     code: str
@@ -2400,6 +2463,7 @@ AskResponseItemType = Union[
     MetadataAskResponseItem,
     AugmentedContextResponseItem,
     CitationsAskResponseItem,
+    FootnoteCitationsAskResponseItem,
     StatusAskResponseItem,
     ErrorAskResponseItem,
     RetrievalAskResponseItem,

{nucliadb_models-6.8.1.post4983.dist-info → nucliadb_models-6.9.5.post5452.dist-info}/METADATA

@@ -1,19 +1,18 @@
 Metadata-Version: 2.4
 Name: nucliadb_models
-Version: 6.8.1.post4983
+Version: 6.9.5.post5452
 Author-email: Nuclia <nucliadb@nuclia.com>
 License-Expression: Apache-2.0
 Project-URL: Homepage, https://nuclia.com
 Project-URL: Repository, https://github.com/nuclia/nucliadb
 Classifier: Development Status :: 4 - Beta
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3 :: Only
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: <4,>=3.9
+Requires-Python: <4,>=3.10
 Description-Content-Type: text/markdown
 Requires-Dist: pydantic!=2.11.5,!=2.11.6,>=2.6
 

{nucliadb_models-6.8.1.post4983.dist-info → nucliadb_models-6.9.5.post5452.dist-info}/RECORD

@@ -1,22 +1,24 @@
 nucliadb_models/__init__.py,sha256=3y8-htogKuCZcbhaUZdSjTeEjUSeec9aRWyL8AlKCyM,1077
-nucliadb_models/common.py,sha256=YW84w1NAQARObs2nXw6YBgdxQJeVCmTZZr5lSqj-IdQ,7904
-nucliadb_models/configuration.py,sha256=aTV5mBwYFlwiV1_nWyVAXaCh7F6lDVTVh28Xfwy8ox8,2448
+nucliadb_models/augment.py,sha256=vAtFh4D4eC4nvfwaRTlfeuAMOL9Z9TFZnUNiRAMasss,2543
+nucliadb_models/common.py,sha256=2dtKG4ZNi9p-yoNY76Uvyu1SlMeNYpH-MnuU3Q6w9Js,8169
+nucliadb_models/configuration.py,sha256=BBrJsNjP324Cw_5J3dBrGwvpkHQYbXEo3TUaI9IqAOg,2449
 nucliadb_models/content_types.py,sha256=36Ga-iGf4ivCqgtXC7imFgegrwHB117s9eqP62JtGv0,3456
-nucliadb_models/conversation.py,sha256=i8tvQxUj6Hw0Nc2oqCGxxzSA1CCY6h7I0ucop9akops,4859
+nucliadb_models/conversation.py,sha256=k9bKhkDiqhqmdrDfDPNoUfG7-2H_-KAyuOnETd8zV0E,5081
 nucliadb_models/entities.py,sha256=i-7Y8qmFRRTih5zw0ajv1U_iiXexe66M3TK8hUikQZk,2356
 nucliadb_models/export_import.py,sha256=mNm9IArOLnC6TLupkwqVFhxD5d08mpIVOVFneECv8UA,1073
-nucliadb_models/external_index_providers.py,sha256=IIKjJjLixWQC1zrbzam2FDcAo5UUxShZfueZSxqZu8Y,1535
+nucliadb_models/external_index_providers.py,sha256=pL3leo4MkuJOnKlU1Sg6GT_mnK_VUBxGui-RPmDYVWU,1126
 nucliadb_models/extracted.py,sha256=Owz7LC3le3Dvau3TtRiO8NY84meOf6IxN-RrOqqpMPs,5593
 nucliadb_models/file.py,sha256=tXtgB9c7i2ADsnJ7HdbXyroAmXadGvOeA49htBh7BZo,2263
 nucliadb_models/filters.py,sha256=NQI2-4AFzzJuZy8NeY3jXlTbbU5wxiwMCP-5DrD-7lE,14759
+nucliadb_models/hydration.py,sha256=SlAzraJE6DX0uOpZWxu2k_9-ikYorsj0t8xwsWSBQZY,14363
 nucliadb_models/labels.py,sha256=9zqRgkpZuX3kUPwsTTgCH7JyOWK7dM5pwyuHJR86YdU,3949
 nucliadb_models/link.py,sha256=PF5hHLwdOed5TMBTxtokkgWtMh1bFnORZjybh0NwVCw,2526
-nucliadb_models/metadata.py,sha256=MFVYnpXMBoY4ylMg029o7yDKGxhK7NB0c0FSshzJHm4,8356
+nucliadb_models/metadata.py,sha256=OOKGy_83NtlG1QKQZEwMuwu4wbVEe7P30Y2QvnGSDto,8933
 nucliadb_models/notifications.py,sha256=mna8-AoD_29Wds0Thl0AF0zpERnJmYGLZX1w1fUopMY,4036
 nucliadb_models/processing.py,sha256=nhKuHQjqCdb9zJVkYGPTLub23tK9e_lwL5OCDVymZjY,719
 nucliadb_models/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 nucliadb_models/resource.py,sha256=RzCos0QRgSMkaV-p7EoceSmt7UTzt9G9be5BKF-iGrQ,9021
-nucliadb_models/search.py,sha256=d1KTOxd2VDKTkrJmwKSqOcZ8lcSXZpULK7a5zbMUS2c,94050
+nucliadb_models/search.py,sha256=_vn3pDXcK4iwiCfim3BtlD5EaQAeXoxl2IfNDsrKesA,97514
 nucliadb_models/security.py,sha256=opxaDLfvk3aU0sjesK0jGrYLx5h4YCwlKKN0moYs_ig,1150
 nucliadb_models/synonyms.py,sha256=afbaVqSQSxGLwi2PusVaLSRpkOtA5AZmWOKd1f4nl2E,690
 nucliadb_models/text.py,sha256=60bxZnOjRHnDdezR8VfR3AZsXTOwePFPs2BKB8wxBak,3414
@@ -32,7 +34,7 @@ nucliadb_models/graph/responses.py,sha256=Sdq8OgFAL1YT-1lJyLLrkqcScvj7YTEqAUwQ-k
 nucliadb_models/internal/__init__.py,sha256=zG33bUz1rHFPtvqQPWn4rDwBJt3FJodGuQYD45quiQg,583
 nucliadb_models/internal/predict.py,sha256=Pnx6MmLfK65eExe1XnVxqmSlvMwdowewwks9BOEoqMw,2029
 nucliadb_models/internal/shards.py,sha256=__y1OZtWGiNcPQEWfSFOj8yw458WGi7mM4vZe0K-L1Y,1691
-nucliadb_models-6.8.1.post4983.dist-info/METADATA,sha256=1_uAWLGHp8_M1Y91Byb5aHUT9b0izZgsecCB9-d4t6Y,794
-nucliadb_models-6.8.1.post4983.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-nucliadb_models-6.8.1.post4983.dist-info/top_level.txt,sha256=UrY1I8oeovIRwkXLYplssTrxQdUjhSEFDFbnwaIV3tA,16
-nucliadb_models-6.8.1.post4983.dist-info/RECORD,,
+nucliadb_models-6.9.5.post5452.dist-info/METADATA,sha256=B4hcUdlQ5Uk9ik2FUjSkT6ZQHyby87Q7eYlGGMx8wkI,745
+nucliadb_models-6.9.5.post5452.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+nucliadb_models-6.9.5.post5452.dist-info/top_level.txt,sha256=UrY1I8oeovIRwkXLYplssTrxQdUjhSEFDFbnwaIV3tA,16
+nucliadb_models-6.9.5.post5452.dist-info/RECORD,,