docling-core 0.0.1__py3-none-any.whl

docling_core/types/nlp/qa_labels.py

@@ -0,0 +1,118 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define models for labeling Q&A pairs."""
+from typing import Literal, Optional
+
+from pydantic import BaseModel, Field
+
+from docling_core.search.mapping import es_field
+
+QAScopeLabel = Literal["corpus", "document", "out_of_scope"]
+QAAlignmentLabel = Literal["aligned", "tangential", "misaligned"]
+QACorrectnessLabel = Literal["entailed", "not_entailed"]
+QACompletenessLabel = Literal["complete", "incomplete"]
+QAInformationLabel = Literal[
+    "fact_single",
+    "fact_multi",
+    "summary",
+    "reasoning",
+    "choice",
+    "procedure",
+    "opinion",
+    "feedback",
+]
+
+
+class QALabelling(BaseModel, extra="forbid"):
+    """Subclass to classify QA pair."""
+
+    scope: Optional[QAScopeLabel] = Field(
+        default=None,
+        description="""Enumeration of QA scope types based on question only.
+            - Corpus: question is asked on the entire corpus
+                > Example: "What is the operating temperature of device X?"
+            - Document: need to know the precise document before answering the question
+                > Example: "What is its operating temperature?"
+            - Out of scope: question is out of scope for the system
+                > Example: "What is the volume of moon?" """,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    alignment: Optional[QAAlignmentLabel] = Field(
+        default=None,
+        description="""Enumeration of QA alignment types based on question-context pair.
+            Given the following context: "Device X works between 2 and 20 degrees C"
+            A question can be:
+            - Aligned: the context has information that the question seeks
+                > Example: "Can device X work at 10 degrees?"
+            - Tangential: the context does not have the information directly
+                            but the question is related to the context
+                > Example: "Is device X safe?"
+            - Misaligned: the question has nothing to do with the context
+                > Example: "Why is device Y not working?" """,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    correctness: Optional[QACorrectnessLabel] = Field(
+        default=None,
+        description="""Enumeration of QA correctness types based on
+            question-answer-context triplet.
+            Given the following context: "Device X works between 2 and 20 degrees C"
+            and the following question: "Can device X work at 10 degrees?"
+            An answer can be:
+            - Entailed: answer is entailed to both question and context
+                > Example: "Yes, as it works between 2 and 20 degrees."
+            - Not entailed: answer is not entailed to either question or context
+                > Example: "Yes, device X can work at any temperature." """,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    completeness: Optional[QACompletenessLabel] = Field(
+        default=None,
+        description="""Enumeration of QA completeness types based on
+            question-answer-context triplet.
+            Given the following context: "A, B, C, and D met on Friday."
+            and the following question: "Who was in the meeting?"
+            An answer can be:
+            - Complete: Answer contains all relevant information requested by a
+                question that can be extracted from the associated ground-truth context
+                > Example: "A, B, C, and D."
+            - Incomplete: Answer does not contain the entire relevant information in
+                the context
+                > Example: "B and D" """,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    information: Optional[QAInformationLabel] = Field(
+        default=None,
+        description="""Enumeration of QA nature of information types based on question
+            only.
+            - Single fact: Answer should be a short phrase containing a numerical or
+                textual fact
+                > Example: "What is the boiling point of water?"
+            - Multiple fact: Answer is a list of two or more facts (not necessarily in
+                list format)
+                > Example: "What is the minimum and maximum age of people working at
+                IBM?"
+            - Summary: Answer summarises a part of the context without any modification.
+                > Example: "Briefly describe the temperature requirements for this
+                device in a table"
+            - Reasoning: Answer requires inferring information from the context that
+                can be inferred but is not explicitly stated (e.g., operating
+                temperature is given and the question asks if the device can operate at
+                a particular temperature)
+                > Example: "Why can I not operate this device under water?"
+            - Multiple choice: Question provides a few choices implicitly or explicitly
+                and the answer must be one of these choices. Includes yes/no questions
+                > Example: "If I operate this device at 10 degrees, will it be in the
+                green range or red?"
+            - Procedure: Answer outlines the steps to do something. As opposed to a
+                summary, the order of information matters here
+                > Example: "How can I access part X of device Y?"
+            - Opinion: The context provides several viewpoints and the question
+                requests the opinion of the chatbot
+                > Example: "Is device X better than Y?"
+            - Feedback: The question is actually a feedback on the preceding generation
+                within a session
+                > Example: "Your summary was inadequate" """,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )

docling_core/types/rec/__init__.py

@@ -0,0 +1,6 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Package for models defined by the Record type."""

docling_core/types/rec/attribute.py

@@ -0,0 +1,55 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the model Attribute."""
+from typing import Generic, Optional
+
+from pydantic import Field
+from typing_extensions import Annotated
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import (
+    IdentifierTypeT,
+    PredicateKeyNameT,
+    PredicateKeyTypeT,
+    PredicateValueTypeT,
+    ProvenanceTypeT,
+)
+from docling_core.types.rec.base import ProvenanceItem
+from docling_core.types.rec.predicate import Predicate
+from docling_core.utils.alias import AliasModel
+
+
+class Attribute(
+    AliasModel,
+    Generic[
+        IdentifierTypeT,
+        PredicateValueTypeT,
+        PredicateKeyNameT,
+        PredicateKeyTypeT,
+        ProvenanceTypeT,
+    ],
+    extra="forbid",
+):
+    """Attribute model that describes a list of characteristics."""
+
+    conf: Annotated[float, Field(strict=True, ge=0.0, le=1.0, allow_inf_nan=False)] = (
+        Field(
+            ...,
+            title="Confidence",
+            description="The confidence level of this attribute characteristics.",
+            json_schema_extra=es_field(type="float"),
+        )
+    )
+
+    prov: Optional[list[ProvenanceItem[IdentifierTypeT, ProvenanceTypeT]]] = Field(
+        default=None,
+        title="Provenance",
+        description="The sources of this attribute characteristics.",
+    )
+
+    predicates: list[
+        Predicate[PredicateValueTypeT, PredicateKeyNameT, PredicateKeyTypeT]
+    ] = Field(..., description="A list of characteristics (type, value, and name).")

docling_core/types/rec/base.py

@@ -0,0 +1,90 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the base models for the Record type."""
+from typing import Generic, List, Optional
+
+from pydantic import Field, StrictInt, StrictStr
+from typing_extensions import Annotated
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import Identifier, IdentifierTypeT, ProvenanceTypeT
+from docling_core.utils.alias import AliasModel
+
+
+class ProvenanceItem(
+    AliasModel, Generic[IdentifierTypeT, ProvenanceTypeT], extra="forbid"
+):
+    """A representation of an object provenance."""
+
+    type_: Optional[ProvenanceTypeT] = Field(
+        default=None,
+        alias="type",
+        title="The provenance type",
+        description=(
+            "Any string representing the type of provenance, e.g. `sentence`, "
+            "`table`, or `doi`."
+        ),
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+
+    text: Optional[StrictStr] = Field(
+        default=None,
+        title="Evidence of the provenance",
+        description=(
+            "A text representing the evidence of the provenance, e.g. the sentence "
+            "text or the content of a table cell"
+        ),
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+
+    reference: Optional[Identifier[IdentifierTypeT]] = Field(
+        default=None,
+        title="Reference to the provenance object",
+        description=(
+            "Reference to another object, e.g. record, statement, URL, or any other "
+            "object that identifies the provenance"
+        ),
+    )
+
+    path: Optional[StrictStr] = Field(
+        default=None,
+        title="The location of the provenance within the referenced object",
+        description=(
+            "A path that locates the evidence within the provenance object identified "
+            "by the `reference` field using a JSON pointer notation, e.g., "
+            "`#/main-text/5` to locate the `main-text` paragraph at index 5"
+        ),
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+
+    span: Optional[Annotated[List[StrictInt], Field(min_length=2, max_length=2)]] = (
+        Field(
+            default=None,
+            title="The location of the item in the text/table",
+            description=(
+                "location of the item in the text/table referenced by the `path`,"
+                " e.g., `[34, 67]`"
+            ),
+        )
+    )
+
+
+class Provenance(AliasModel, Generic[IdentifierTypeT, ProvenanceTypeT]):
+    """A representation of an evidence, as a list of provenance objects."""
+
+    conf: Annotated[float, Field(strict=True, ge=0.0, le=1.0)] = Field(
+        ...,
+        title="The confidence of the evidence",
+        description=(
+            "This value represents a score to the data item. Items originating from "
+            " databases will typically have a score 1.0, while items resulting from "
+            " an NLP model may have a value between 0.0 and 1.0."
+        ),
+        json_schema_extra=es_field(type="float"),
+    )
+    prov: list[ProvenanceItem[IdentifierTypeT, ProvenanceTypeT]] = Field(
+        title="Provenance", description="A list of provenance items."
+    )

docling_core/types/rec/predicate.py

@@ -0,0 +1,133 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the model Predicate."""
+from datetime import datetime
+from typing import Annotated, Generic, Optional
+
+from pydantic import (
+    BaseModel,
+    Field,
+    StrictBool,
+    StrictFloat,
+    StrictStr,
+    field_validator,
+)
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import (
+    Coordinates,
+    PredicateKeyNameT,
+    PredicateKeyTypeT,
+    PredicateValueTypeT,
+)
+from docling_core.utils.alias import AliasModel
+
+
+class NumericalValue(BaseModel, extra="forbid"):
+    """Model for numerical values."""
+
+    min: StrictFloat = Field(..., json_schema_extra=es_field(type="float"))
+    max: StrictFloat = Field(..., json_schema_extra=es_field(type="float"))
+    val: StrictFloat = Field(..., json_schema_extra=es_field(type="float"))
+    err: StrictFloat = Field(..., json_schema_extra=es_field(type="float"))
+    unit: StrictStr = Field(
+        ..., json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+
+
+class NominalValue(BaseModel, extra="forbid"):
+    """Model for nominal (categorical) values."""
+
+    value: StrictStr = Field(
+        ..., json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+
+
+class TextValue(BaseModel, extra="forbid"):
+    """Model for textual values."""
+
+    value: StrictStr = Field(..., json_schema_extra=es_field(type="text"))
+
+
+class BooleanValue(BaseModel, extra="forbid"):
+    """Model for boolean values."""
+
+    value: StrictBool = Field(..., json_schema_extra=es_field(type="boolean"))
+
+
+class DatetimeValue(BaseModel, extra="forbid"):
+    """Model for datetime values."""
+
+    value: datetime
+
+
+class GeopointValue(BaseModel, extra="forbid"):
+    """A representation of a geopoint (longitude and latitude coordinates)."""
+
+    value: Coordinates
+    conf: Optional[Annotated[float, Field(strict=True, ge=0.0, le=1.0)]] = Field(
+        default=None, json_schema_extra=es_field(type="float")
+    )
+
+    @field_validator("value")
+    @classmethod
+    def validate_coordinates(cls, v):
+        """Validate the reference field for indexes of type Document."""
+        if abs(v[0]) > 180:
+            raise ValueError("invalid longitude")
+        if abs(v[1]) > 90:
+            raise ValueError("invalid latitude")
+        return v
+
+
+class PredicateKey(
+    AliasModel, Generic[PredicateKeyNameT, PredicateKeyTypeT], extra="forbid"
+):
+    """Model for the key (unique identifier) of a predicate."""
+
+    name: PredicateKeyNameT = Field(
+        description="Name of the predicate key.",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    type_: PredicateKeyTypeT = Field(
+        alias="type",
+        title="Type",
+        description="Type of predicate key.",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+
+
+class PredicateValue(AliasModel, Generic[PredicateValueTypeT], extra="forbid"):
+    """Model for the value of a predicate."""
+
+    name: StrictStr = Field(
+        description="Name of the predicate value (actual value).",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    type_: PredicateValueTypeT = Field(
+        alias="type",
+        description="Type of predicate value.",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+
+
+class Predicate(
+    AliasModel,
+    Generic[PredicateValueTypeT, PredicateKeyNameT, PredicateKeyTypeT],
+    extra="forbid",
+):
+    """Model for a predicate."""
+
+    key: PredicateKey[PredicateKeyNameT, PredicateKeyTypeT]
+    value: PredicateValue[PredicateValueTypeT]
+
+    numerical_value: Optional[NumericalValue] = None
+    numerical_value_si: Optional[NumericalValue] = None
+    nominal_value: Optional[NominalValue] = None
+    text_value: Optional[TextValue] = None
+    boolean_value: Optional[BooleanValue] = None
+    datetime_value: Optional[DatetimeValue] = None
+    geopoint_value: Optional[GeopointValue] = None

docling_core/types/rec/record.py

@@ -0,0 +1,95 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the model Record."""
+from typing import Generic, Optional
+
+from pydantic import BaseModel, Field, StrictStr
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import (
+    Acquisition,
+    CollectionNameTypeT,
+    CollectionRecordInfo,
+    FileInfoObject,
+    Identifier,
+    IdentifierTypeT,
+    Log,
+    PredicateKeyNameT,
+    PredicateKeyTypeT,
+    PredicateValueTypeT,
+    StrictDateTime,
+    SubjectNameTypeT,
+    SubjectTypeT,
+)
+from docling_core.types.rec.attribute import Attribute
+from docling_core.types.rec.base import Provenance, ProvenanceTypeT
+from docling_core.types.rec.subject import Subject
+
+
+class RecordDescription(BaseModel, Generic[CollectionNameTypeT]):
+    """Additional record metadata, including optional collection-specific fields."""
+
+    logs: list[Log] = Field(
+        description="Logs that describe the ETL tasks applied to this record."
+    )
+    publication_date: Optional[StrictDateTime] = Field(
+        default=None,
+        title="Publication date",
+        description=(
+            "The date that best represents the last publication time of a record."
+        ),
+    )
+    collection: Optional[CollectionRecordInfo[CollectionNameTypeT]] = Field(
+        default=None, description="The collection information of this record."
+    )
+    acquisition: Optional[Acquisition] = Field(
+        default=None,
+        description=(
+            "Information on how the document was obtained, for data governance"
+            " purposes."
+        ),
+    )
+
+
+class Record(
+    Provenance,
+    Generic[
+        IdentifierTypeT,
+        PredicateValueTypeT,
+        PredicateKeyNameT,
+        PredicateKeyTypeT,
+        ProvenanceTypeT,
+        SubjectTypeT,
+        SubjectNameTypeT,
+        CollectionNameTypeT,
+    ],
+):
+    """A representation of a structured record in an database."""
+
+    file_info: FileInfoObject = Field(alias="file-info")
+    description: RecordDescription
+    subject: Subject[IdentifierTypeT, SubjectTypeT, SubjectNameTypeT]
+    attributes: Optional[
+        list[
+            Attribute[
+                IdentifierTypeT,
+                PredicateValueTypeT,
+                PredicateKeyNameT,
+                PredicateKeyTypeT,
+                ProvenanceTypeT,
+            ]
+        ]
+    ] = None
+    name: Optional[StrictStr] = Field(
+        default=None,
+        description="A short description or summary of the record.",
+        alias="_name",
+        json_schema_extra=es_field(type="text"),
+    )
+    identifiers: Optional[list[Identifier[IdentifierTypeT]]] = Field(
+        default=None,
+        description="A list of unique identifiers of this record in a database.",
+    )

docling_core/types/rec/statement.py

@@ -0,0 +1,41 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the model Statement."""
+from typing import Generic
+
+from pydantic import Field
+
+from docling_core.types.base import (
+    IdentifierTypeT,
+    PredicateKeyNameT,
+    PredicateKeyTypeT,
+    PredicateValueTypeT,
+    ProvenanceTypeT,
+    SubjectNameTypeT,
+    SubjectTypeT,
+)
+from docling_core.types.rec.attribute import Attribute
+from docling_core.types.rec.subject import Subject
+
+
+class Statement(
+    Attribute,
+    Generic[
+        IdentifierTypeT,
+        PredicateValueTypeT,
+        PredicateKeyNameT,
+        PredicateKeyTypeT,
+        ProvenanceTypeT,
+        SubjectTypeT,
+        SubjectNameTypeT,
+    ],
+    extra="allow",
+):
+    """A representation of a statement on a subject."""
+
+    subject: Subject[IdentifierTypeT, SubjectTypeT, SubjectNameTypeT] = Field(
+        description="The subject (entity) of this statement."
+    )

docling_core/types/rec/subject.py

@@ -0,0 +1,77 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the model Subject."""
+from typing import Generic, Optional
+
+from pydantic import Field, StrictStr
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import (
+    Identifier,
+    IdentifierTypeT,
+    SubjectNameTypeT,
+    SubjectTypeT,
+)
+from docling_core.types.doc.base import S3Reference
+from docling_core.utils.alias import AliasModel
+
+
+class SubjectNameIdentifier(Identifier[SubjectNameTypeT], Generic[SubjectNameTypeT]):
+    """Identifier of subject names.""" ""
+
+
+class Subject(
+    AliasModel,
+    Generic[IdentifierTypeT, SubjectTypeT, SubjectNameTypeT],
+    extra="forbid",
+):
+    """A representation of a subject."""
+
+    display_name: StrictStr = Field(
+        title="Display Name",
+        description=(
+            "Name of the subject in natural language. It can be used for end-user "
+            "applications to display a human-readable name. For instance, `B(2) Mg(1)` "
+            "for `MgB2` or `International Business Machines` for `IBM`"
+        ),
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    display_image: Optional[S3Reference] = Field(
+        default=None,
+        title="Display Image",
+        description=(
+            "Image representing the subject. It can be used for end-user applications."
+            "For example, the chemical structure drawing of a compound "
+            "or the eight bar IBM logo for IBM."
+        ),
+        json_schema_extra=es_field(suppress=True),
+    )
+    type_: SubjectTypeT = Field(
+        alias="type",
+        description=(
+            "Main subject type. For instance, `material`, `material-class`, "
+            "`material-device`, `company`, or `person`."
+        ),
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    names: list[SubjectNameIdentifier[SubjectNameTypeT]] = Field(
+        description=(
+            "List of given names for this subject. They may not be unique across "
+            "different subjects."
+        )
+    )
+    identifiers: Optional[list[Identifier[IdentifierTypeT]]] = Field(
+        default=None,
+        description=(
+            "List of unique identifiers in database. For instance, the `PubChem ID` "
+            "of a record in the PubChem database."
+        ),
+    )
+    labels: Optional[list[StrictStr]] = Field(
+        default=None,
+        description="List of labels or categories for this subject.",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )

docling_core/utils/__init__.py

@@ -0,0 +1,6 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Package for modules to support data models."""

docling_core/utils/alias.py

@@ -0,0 +1,27 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define utility models and types related to field aliases."""
+from pydantic import BaseModel, ConfigDict
+
+
+class AliasModel(BaseModel):
+    """Model for alias fields to ensure instantiation and serialization by alias."""
+
+    model_config = ConfigDict(populate_by_name=True)
+
+    def model_dump(self, **kwargs) -> dict:
+        """Generate a dictionary representation of the model using field aliases."""
+        if "by_alias" not in kwargs:
+            kwargs = {**kwargs, "by_alias": True}
+
+        return super().model_dump(**kwargs)
+
+    def model_dump_json(self, **kwargs) -> str:
+        """Generate a JSON representation of the model using field aliases."""
+        if "by_alias" not in kwargs:
+            kwargs = {**kwargs, "by_alias": True}
+
+        return super().model_dump_json(**kwargs)