docling-core 0.0.1__py3-none-any.whl

docling_core/types/doc/doc_raw.py

@@ -0,0 +1,187 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Models for CCS objects in raw format."""
+from typing import Any, List, Optional
+
+from pydantic import BaseModel, Field
+from typing_extensions import Annotated
+
+from docling_core.types.doc.base import BoundingBox
+from docling_core.utils.alias import AliasModel
+
+FontDifferences = dict[str, Any]
+NamedWidths = dict[str, Any]
+IgnoredCell = Any
+
+
+class Box(BaseModel):
+    """Box."""
+
+    baseline: BoundingBox
+    device: BoundingBox
+
+
+class Content(BaseModel):
+    """Content."""
+
+    rnormalized: str
+
+
+class Enumeration(BaseModel):
+    """Enumeration."""
+
+    match: int
+    type: int
+
+
+class Font(BaseModel):
+    """Font."""
+
+    color: Annotated[List[float], Field(min_length=3, max_length=4)]
+    name: str
+    size: float
+
+
+class Cell(AliasModel):
+    """Cell."""
+
+    see_cell: bool = Field(..., alias="SEE_cell")
+    see_confidence: float = Field(..., alias="SEE_confidence")
+    angle: float
+    box: Box
+    content: Content
+    enumeration: Enumeration
+    font: Font
+
+
+class PageDimensions(BaseModel):
+    """PageDimensions."""
+
+    bbox: BoundingBox
+    height: float
+    width: float
+
+
+class Path(AliasModel):
+    """Path."""
+
+    bbox: BoundingBox
+    sub_paths: list[float] = Field(..., alias="sub-paths")
+    type: str
+    x_values: list[float] = Field(..., alias="x-values")
+    y_values: list[float] = Field(..., alias="y-values")
+
+
+class VerticalLine(BaseModel):
+    """Vertical line."""
+
+    y0: int
+    y1: int
+    x: int
+
+
+class HorizontalLine(BaseModel):
+    """Horizontal line."""
+
+    x0: int
+    x1: int
+    y: int
+
+
+class Image(BaseModel):
+    """Image."""
+
+    box: BoundingBox
+    height: float
+    width: float
+
+
+class FontRange(BaseModel):
+    """Font range."""
+
+    first: int
+    second: int
+
+
+class FontCmap(BaseModel):
+    """Font cmap."""
+
+    cmap: dict[str, str]
+    name: str
+    range: FontRange
+    type: int
+
+
+class FontMetrics(AliasModel):
+    """Font metrics."""
+
+    stem_h: float = Field(..., alias="StemH")
+    stem_v: float = Field(..., alias="StemV")
+    ascent: float
+    average_width: float = Field(..., alias="average-width")
+    bbox: BoundingBox
+    cap_height: float
+    default_width: float = Field(..., alias="default-width")
+    descent: float
+    file: str
+    italic_angle: float = Field(..., alias="italic-angle")
+    max_width: float = Field(..., alias="max-width")
+    missing_width: float = Field(..., alias="missing-width")
+    name: str
+    named_widths: NamedWidths = Field(..., alias="named-widths")
+    weight: str
+    widths: dict[str, float]
+    x_height: float
+
+
+class FontInfo(AliasModel):
+    """Font info."""
+
+    font_cmap: FontCmap = Field(..., alias="font-cmap")
+    font_differences: FontDifferences = Field(..., alias="font-differences")
+    font_metrics: FontMetrics = Field(..., alias="font-metrics")
+    name: str
+    internal_name: str = Field(..., alias="name (internal)")
+    subtype: str
+
+
+class Page(AliasModel):
+    """Page."""
+
+    height: float
+    width: float
+    dimensions: PageDimensions
+    cells: list[Cell]
+    paths: list[Path]
+    vertical_lines: Optional[list[VerticalLine]] = Field(..., alias="vertical-lines")
+    horizontal_lines: Optional[list[HorizontalLine]] = Field(
+        ..., alias="horizontal-lines"
+    )
+    ignored_cells: list[IgnoredCell] = Field(..., alias="ignored-cells")
+    images: list[Image]
+    fonts: dict[str, FontInfo]
+
+
+class Histograms(AliasModel):
+    """Histogram."""
+
+    mean_char_height: dict[str, float] = Field(..., alias="mean-char-height")
+    mean_char_width: dict[str, float] = Field(..., alias="mean-char-width")
+    number_of_chars: dict[str, int] = Field(..., alias="number-of-chars")
+
+
+class PdfInfo(BaseModel):
+    """PDF info."""
+
+    histograms: Histograms
+    styles: list[str]
+
+
+class RawPdf(BaseModel):
+    """Raw PDF."""
+
+    info: PdfInfo
+    pages: list[Page]

docling_core/types/doc/document.py

@@ -0,0 +1,393 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Models for the Docling Document data type."""
+
+from datetime import datetime
+from typing import Generic, Optional, Union
+
+from pydantic import (
+    AnyHttpUrl,
+    BaseModel,
+    Field,
+    NonNegativeInt,
+    StrictStr,
+    model_validator,
+)
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import (
+    Acquisition,
+    CollectionDocumentInfo,
+    CollectionNameTypeT,
+    DescriptionAdvancedT,
+    DescriptionAnalyticsT,
+    FileInfoObject,
+    Identifier,
+    IdentifierTypeT,
+    LanguageT,
+    Log,
+)
+from docling_core.types.doc.base import (
+    BaseCell,
+    BaseText,
+    BitmapObject,
+    PageDimensions,
+    PageReference,
+    Ref,
+    S3Data,
+    Table,
+)
+from docling_core.utils.alias import AliasModel
+
+
+class CCSFileInfoDescription(BaseModel, extra="forbid"):
+    """File info description."""
+
+    author: Optional[list[StrictStr]] = None
+    keywords: Optional[str] = None
+    subject: Optional[str] = None
+    title: Optional[StrictStr] = None
+    creation_date: Optional[str] = None  # datetime
+
+
+class CCSFileInfoObject(FileInfoObject, extra="forbid"):
+    """File info object."""
+
+    num_pages: Optional[int] = Field(default=None, alias="#-pages")
+
+    collection_name: Optional[str] = Field(
+        default=None,
+        alias="collection-name",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    description: Optional[CCSFileInfoDescription] = Field(
+        default=None, json_schema_extra=es_field(suppress=True)
+    )
+    page_hashes: Optional[list[PageReference]] = Field(
+        default=None, alias="page-hashes"
+    )
+
+
+class Affiliation(BaseModel, extra="forbid"):
+    """Affiliation."""
+
+    name: str = Field(
+        ...,
+        json_schema_extra=es_field(
+            fields={
+                "lower": {
+                    "normalizer": "lowercase_asciifolding",
+                    "type": "keyword",
+                    "ignore_above": 8191,
+                },
+                "keyword": {"type": "keyword", "ignore_above": 8191},
+            },
+        ),
+    )
+    id: Optional[str] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    source: Optional[str] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+
+
+class Author(BaseModel, extra="forbid"):
+    """Author."""
+
+    name: str = Field(
+        ...,
+        json_schema_extra=es_field(
+            type="text",
+            fields={
+                "lower": {
+                    "normalizer": "lowercase_asciifolding",
+                    "type": "keyword",
+                    "ignore_above": 8191,
+                },
+                "keyword": {"type": "keyword", "ignore_above": 8191},
+            },
+        ),
+    )
+    id: Optional[str] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    source: Optional[str] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    affiliations: Optional[list[Affiliation]] = None
+
+
+class Publication(BaseModel, Generic[IdentifierTypeT], extra="forbid"):
+    """Publication details of a journal or venue."""
+
+    identifiers: Optional[list[Identifier[IdentifierTypeT]]] = Field(
+        default=None,
+        description="Unique identifiers of a publication venue.",
+    )
+    name: StrictStr = Field(
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+        description="Name of the publication.",
+    )
+    alternate_names: Optional[list[StrictStr]] = Field(
+        default=None,
+        json_schema_extra=es_field(type="text"),
+        title="Alternate Names",
+        description="Other names or abbreviations of this publication.",
+    )
+    type: Optional[list[StrictStr]] = Field(
+        default=None,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+        description="Type of publication (journal article, conference, review,...).",
+    )
+    pages: Optional[StrictStr] = Field(
+        default=None,
+        json_schema_extra=es_field(type="text"),
+        description="Page range in the publication.",
+    )
+    issue: Optional[StrictStr] = Field(
+        default=None,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+        description="Publication issue (issue number).",
+    )
+    volume: Optional[StrictStr] = Field(
+        default=None,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+        description="Publication volume.",
+    )
+    url: Optional[AnyHttpUrl] = Field(
+        default=None,
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+        description="URL on the publication site.",
+    )
+
+
+class DescriptionLicense(BaseModel, extra="forbid"):
+    """Licence in document description."""
+
+    code: Optional[StrictStr] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    text: Optional[StrictStr] = None
+
+
+class CCSDocumentDescription(
+    AliasModel,
+    Generic[
+        DescriptionAdvancedT,
+        DescriptionAnalyticsT,
+        IdentifierTypeT,
+        LanguageT,
+        CollectionNameTypeT,
+    ],
+):
+    """Description in document."""
+
+    title: Optional[StrictStr] = None
+    abstract: Optional[list[StrictStr]] = None
+    authors: Optional[list[Author]] = None
+    affiliations: Optional[list[Affiliation]] = None
+    subjects: Optional[list[str]] = Field(
+        default=None,
+        json_schema_extra=es_field(
+            fields={"keyword": {"ignore_above": 8191, "type": "keyword"}}
+        ),
+    )
+    keywords: Optional[list[str]] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    publication_date: Optional[datetime] = None
+    languages: Optional[list[LanguageT]] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    license_: Optional[DescriptionLicense] = Field(default=None, alias="license")
+    publishers: Optional[list[StrictStr]] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    url_refs: Optional[list[str]] = Field(
+        default=None, json_schema_extra=es_field(type="keyword", ignore_above=8191)
+    )
+    references: Optional[list[Identifier[IdentifierTypeT]]] = None
+    publication: Optional[list[Publication]] = Field(
+        default=None, description="List of publication journals or venues."
+    )
+    reference_count: Optional[NonNegativeInt] = Field(
+        default=None,
+        title="Reference Count",
+        description="Total number of documents referenced by this document.",
+        json_schema_extra=es_field(type="integer"),
+    )
+    citation_count: Optional[NonNegativeInt] = Field(
+        default=None,
+        title="Citation Count",
+        description=(
+            "Total number of citations that this document has received (number "
+            "of documents in whose bibliography this document appears)."
+        ),
+        json_schema_extra=es_field(type="integer"),
+    )
+    citation_date: Optional[datetime] = Field(
+        default=None,
+        title="Citation Count Date",
+        description="Last update date of the citation count.",
+    )
+    advanced: Optional[DescriptionAdvancedT] = None
+    analytics: Optional[DescriptionAnalyticsT] = None
+    logs: list[Log]
+    collection: Optional[CollectionDocumentInfo[CollectionNameTypeT]] = Field(
+        default=None, description="The collection information of this document."
+    )
+    acquisition: Optional[Acquisition] = Field(
+        default=None,
+        description=(
+            "Information on how the document was obtained, for data governance"
+            " purposes."
+        ),
+    )
+
+
+class MinimalDocument(
+    AliasModel,
+    Generic[
+        DescriptionAdvancedT,
+        DescriptionAnalyticsT,
+        IdentifierTypeT,
+        LanguageT,
+        CollectionNameTypeT,
+    ],
+):
+    """Minimal model for a document."""
+
+    name: StrictStr = Field(alias="_name")
+    obj_type: StrictStr = Field("document", alias="type")
+    description: CCSDocumentDescription[
+        DescriptionAdvancedT,
+        DescriptionAnalyticsT,
+        IdentifierTypeT,
+        LanguageT,
+        CollectionNameTypeT,
+    ]
+    file_info: FileInfoObject = Field(alias="file-info")
+    main_text: Optional[list[Union[Ref, BaseText]]] = Field(
+        default=None, alias="main-text"
+    )
+    figures: Optional[list[BaseCell]] = None
+    tables: Optional[list[Table]] = None
+
+
+class CCSDocument(
+    MinimalDocument,
+    Generic[
+        DescriptionAdvancedT,
+        DescriptionAnalyticsT,
+        IdentifierTypeT,
+        LanguageT,
+        CollectionNameTypeT,
+    ],
+):
+    """Model for a CCS-generated document."""
+
+    obj_type: StrictStr = Field("pdf-document", alias="type")
+    bitmaps: Optional[list[BitmapObject]] = None
+    equations: Optional[list[BaseCell]] = None
+    footnotes: Optional[list[BaseText]] = None
+    file_info: CCSFileInfoObject = Field(alias="file-info")
+    main_text: Optional[list[Union[Ref, BaseText]]] = Field(
+        default=None,
+        alias="main-text",
+    )
+    page_dimensions: Optional[list[PageDimensions]] = Field(
+        default=None, alias="page-dimensions"
+    )
+    page_footers: Optional[list[BaseText]] = Field(default=None, alias="page-footers")
+    page_headers: Optional[list[BaseText]] = Field(default=None, alias="page-headers")
+    s3_data: Optional[S3Data] = Field(default=None, alias="_s3_data")
+
+    @model_validator(mode="before")
+    @classmethod
+    def from_dict(cls, data):
+        """Validates and fixes the input data."""
+        description_collection = data["description"].get("collection")
+        if not description_collection:
+            data["description"].setdefault("collection", {})
+
+        data["description"]["collection"].setdefault("type", "Document")
+        logs = data["description"].get("logs")
+        if not logs:
+            data["description"].setdefault("logs", [])
+
+        abstract = data["description"].get("abstract")
+        if abstract is not None and not isinstance(abstract, list):
+            if isinstance(abstract, str):
+                data["description"]["abstract"] = [abstract]
+            else:
+                data["description"].pop("abstract")
+
+        for key in ["affiliations", "authors"]:
+            descr = data["description"].get(key)
+            if descr is not None and not isinstance(descr, list):
+                if isinstance(descr, dict):
+                    data["description"][key] = [descr]
+                else:
+                    data["description"].pop(key)
+
+        if data.get("main-text"):
+            for item in data["main-text"]:
+                if ref := item.pop("__ref", None):
+                    item["$ref"] = ref
+
+        return data
+
+
+class ExportedCCSDocument(
+    MinimalDocument,
+    Generic[
+        DescriptionAdvancedT,
+        DescriptionAnalyticsT,
+        IdentifierTypeT,
+        LanguageT,
+        CollectionNameTypeT,
+    ],
+):
+    """Document model for Docling."""
+
+    obj_type: StrictStr = Field(
+        "pdf-document",
+        alias="type",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    bitmaps: Optional[list[BitmapObject]] = None
+    equations: Optional[list[BaseCell]] = None
+    footnotes: Optional[list[BaseText]] = None
+    description: CCSDocumentDescription[
+        DescriptionAdvancedT,
+        DescriptionAnalyticsT,
+        IdentifierTypeT,
+        LanguageT,
+        CollectionNameTypeT,
+    ]
+    file_info: CCSFileInfoObject = Field(alias="file-info")
+    main_text: Optional[list[Union[Ref, BaseText]]] = Field(
+        default=None, alias="main-text"
+    )
+    page_dimensions: Optional[list[PageDimensions]] = Field(
+        default=None, alias="page-dimensions"
+    )
+    page_footers: Optional[list[BaseText]] = Field(default=None, alias="page-footers")
+    page_headers: Optional[list[BaseText]] = Field(default=None, alias="page-headers")
+    s3_data: Optional[S3Data] = Field(default=None, alias="_s3_data")
+    identifiers: Optional[list[Identifier[IdentifierTypeT]]] = None
+
+    @model_validator(mode="before")
+    @classmethod
+    def from_dict(cls, data):
+        """Fix ref in main-text."""
+        if data.get("main-text"):
+            for item in data["main-text"]:
+                if ref := item.pop("__ref", None):
+                    item["$ref"] = ref
+
+        return data

docling_core/types/gen/__init__.py

@@ -0,0 +1,6 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Package for models defined by the Generic type."""

docling_core/types/gen/generic.py

@@ -0,0 +1,33 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define a generic Docling type."""
+
+from typing import Optional
+
+from pydantic import Field, StrictStr
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import FileInfoObject
+from docling_core.utils.alias import AliasModel
+
+
+class Generic(AliasModel):
+    """A representation of a generic document."""
+
+    name: Optional[StrictStr] = Field(
+        default=None,
+        description="A short description or summary of the document.",
+        alias="_name",
+        json_schema_extra=es_field(type="text"),
+    )
+
+    file_info: FileInfoObject = Field(
+        title="Document information",
+        description=(
+            "Minimal identification information of the document within a collection."
+        ),
+        alias="file-info",
+    )

docling_core/types/nlp/__init__.py

@@ -0,0 +1,6 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Package for models defining NLP artifacts."""

docling_core/types/nlp/qa.py

@@ -0,0 +1,74 @@
+#
+# Copyright IBM Corp. 2024 - 2024
+# SPDX-License-Identifier: MIT
+#
+
+"""Define the model for Q&A pairs."""
+from typing import Generic, Optional
+
+from pydantic import BaseModel, Field, StrictBool, StrictStr
+
+from docling_core.search.mapping import es_field
+from docling_core.types.base import DescriptionAdvancedT, StrictDateTime, UniqueList
+from docling_core.types.nlp.qa_labels import QALabelling
+
+
+class QAPair(BaseModel, Generic[DescriptionAdvancedT]):
+    """A representation of a question-answering (QA) pair."""
+
+    context: StrictStr = Field(
+        description=(
+            "A single string containing the context of the question enabling the"
+            " presentation of the answer."
+        )
+    )
+    question: StrictStr = Field(description="A question on the given context.")
+    answer: StrictStr = Field(
+        description="The answer to the question from the context."
+    )
+    short_answer: Optional[StrictStr] = Field(
+        default=None, description="Alternative and concise answer."
+    )
+    retrieved_context: Optional[StrictBool] = Field(
+        default=False,
+        description="Whether the context was retrieved from the question.",
+    )
+    generated_question: Optional[StrictBool] = Field(
+        default=False, description="Whether the question was generated by an AI model."
+    )
+    generated_answer: Optional[StrictBool] = Field(
+        default=False, description="Whether the answer was generated by an AI model."
+    )
+    created: StrictDateTime = Field(
+        description="Datetime when the QA pair was created ."
+    )
+    user: Optional[StrictStr] = Field(
+        default=None,
+        description=(
+            "Unique identifier of the user that created or curated this QA pair."
+        ),
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    model: Optional[StrictStr] = Field(
+        default=None,
+        description="Unique identifier of the model used to generate this QA pair.",
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    paths: UniqueList[StrictStr] = Field(
+        description=(
+            "One or more references to a document that identify the provenance of the"
+            " QA pair context."
+        ),
+        examples=[
+            "badce7c84d0ba7ba0fb5e94492b0d91e2506a7cb48e4524ad572c546a35f768e#/"
+            "main-text/4"
+        ],
+        json_schema_extra=es_field(type="keyword", ignore_above=8191),
+    )
+    advanced: Optional[DescriptionAdvancedT] = Field(
+        default=None,
+        description="Document metadata to provide more details on the context.",
+    )
+    labels: Optional[QALabelling] = Field(
+        default=None, description="QApair labelling axes."
+    )