docling 2.20.0__tar.gz → 2.22.0__tar.gz

{docling-2.20.0 → docling-2.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling
-Version: 2.20.0
+Version: 2.22.0
 Summary: SDK and CLI for parsing PDF, DOCX, HTML, and more, to a unified document representation for powering downstream workflows such as gen AI applications.
 Home-page: https://github.com/DS4SD/docling
 License: MIT
@@ -28,7 +28,7 @@ Provides-Extra: vlm
 Requires-Dist: beautifulsoup4 (>=4.12.3,<4.13.0)
 Requires-Dist: certifi (>=2024.7.4)
 Requires-Dist: deepsearch-glm (>=1.0.0,<2.0.0)
-Requires-Dist: docling-core[chunking] (>=2.17.2,<3.0.0)
+Requires-Dist: docling-core[chunking] (>=2.18.0,<3.0.0)
 Requires-Dist: docling-ibm-models (>=3.3.0,<4.0.0)
 Requires-Dist: docling-parse (>=3.3.0,<4.0.0)
 Requires-Dist: easyocr (>=1.7,<2.0)
@@ -41,7 +41,7 @@ Requires-Dist: onnxruntime (>=1.7.0,<1.20.0) ; (python_version < "3.10") and (ex
 Requires-Dist: onnxruntime (>=1.7.0,<2.0.0) ; (python_version >= "3.10") and (extra == "rapidocr")
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: pandas (>=2.1.4,<3.0.0)
-Requires-Dist: pillow (>=10.0.0,<11.0.0)
+Requires-Dist: pillow (>=10.0.0,<12.0.0)
 Requires-Dist: pydantic (>=2.0.0,<3.0.0)
 Requires-Dist: pydantic-settings (>=2.3.0,<3.0.0)
 Requires-Dist: pypdfium2 (>=4.30.0,<5.0.0)

docling-2.22.0/docling/backend/csv_backend.py

@@ -0,0 +1,125 @@
+import csv
+import logging
+import warnings
+from io import BytesIO, StringIO
+from pathlib import Path
+from typing import Set, Union
+
+from docling_core.types.doc import DoclingDocument, DocumentOrigin, TableCell, TableData
+
+from docling.backend.abstract_backend import DeclarativeDocumentBackend
+from docling.datamodel.base_models import InputFormat
+from docling.datamodel.document import InputDocument
+
+_log = logging.getLogger(__name__)
+
+
+class CsvDocumentBackend(DeclarativeDocumentBackend):
+    content: StringIO
+
+    def __init__(self, in_doc: "InputDocument", path_or_stream: Union[BytesIO, Path]):
+        super().__init__(in_doc, path_or_stream)
+
+        # Load content
+        try:
+            if isinstance(self.path_or_stream, BytesIO):
+                self.content = StringIO(self.path_or_stream.getvalue().decode("utf-8"))
+            elif isinstance(self.path_or_stream, Path):
+                self.content = StringIO(self.path_or_stream.read_text("utf-8"))
+            self.valid = True
+        except Exception as e:
+            raise RuntimeError(
+                f"CsvDocumentBackend could not load document with hash {self.document_hash}"
+            ) from e
+        return
+
+    def is_valid(self) -> bool:
+        return self.valid
+
+    @classmethod
+    def supports_pagination(cls) -> bool:
+        return False
+
+    def unload(self):
+        if isinstance(self.path_or_stream, BytesIO):
+            self.path_or_stream.close()
+        self.path_or_stream = None
+
+    @classmethod
+    def supported_formats(cls) -> Set[InputFormat]:
+        return {InputFormat.CSV}
+
+    def convert(self) -> DoclingDocument:
+        """
+        Parses the CSV data into a structured document model.
+        """
+
+        # Detect CSV dialect
+        head = self.content.readline()
+        dialect = csv.Sniffer().sniff(head, ",;\t|:")
+        _log.info(f'Parsing CSV with delimiter: "{dialect.delimiter}"')
+        if not dialect.delimiter in {",", ";", "\t", "|", ":"}:
+            raise RuntimeError(
+                f"Cannot convert csv with unknown delimiter {dialect.delimiter}."
+            )
+
+        # Parce CSV
+        self.content.seek(0)
+        result = csv.reader(self.content, dialect=dialect, strict=True)
+        self.csv_data = list(result)
+        _log.info(f"Detected {len(self.csv_data)} lines")
+
+        # Ensure uniform column length
+        expected_length = len(self.csv_data[0])
+        is_uniform = all(len(row) == expected_length for row in self.csv_data)
+        if not is_uniform:
+            warnings.warn(
+                f"Inconsistent column lengths detected in CSV data. "
+                f"Expected {expected_length} columns, but found rows with varying lengths. "
+                f"Ensure all rows have the same number of columns."
+            )
+
+        # Parse the CSV into a structured document model
+        origin = DocumentOrigin(
+            filename=self.file.name or "file.csv",
+            mimetype="text/csv",
+            binary_hash=self.document_hash,
+        )
+
+        doc = DoclingDocument(name=self.file.stem or "file.csv", origin=origin)
+
+        if self.is_valid():
+            # Convert CSV data to table
+            if self.csv_data:
+                num_rows = len(self.csv_data)
+                num_cols = max(len(row) for row in self.csv_data)
+
+                table_data = TableData(
+                    num_rows=num_rows,
+                    num_cols=num_cols,
+                    table_cells=[],
+                )
+
+                # Convert each cell to TableCell
+                for row_idx, row in enumerate(self.csv_data):
+                    for col_idx, cell_value in enumerate(row):
+                        cell = TableCell(
+                            text=str(cell_value),
+                            row_span=1,  # CSV doesn't support merged cells
+                            col_span=1,
+                            start_row_offset_idx=row_idx,
+                            end_row_offset_idx=row_idx + 1,
+                            start_col_offset_idx=col_idx,
+                            end_col_offset_idx=col_idx + 1,
+                            col_header=row_idx == 0,  # First row as header
+                            row_header=False,
+                        )
+                        table_data.table_cells.append(cell)
+
+                doc.add_table(data=table_data)
+        else:
+            raise RuntimeError(
+                f"Cannot convert doc with {self.document_hash} because the backend failed to init."
+            )
+
+        return doc

{docling-2.20.0 → docling-2.22.0}/docling/cli/main.py

@@ -234,6 +234,12 @@ def convert(
         Optional[Path],
         typer.Option(..., help="If provided, the location of the model artifacts."),
     ] = None,
+    enable_remote_services: Annotated[
+        bool,
+        typer.Option(
+            ..., help="Must be enabled when using models connecting to remote services."
+        ),
+    ] = False,
     abort_on_error: Annotated[
         bool,
         typer.Option(
@@ -380,6 +386,7 @@ def convert(
 
         accelerator_options = AcceleratorOptions(num_threads=num_threads, device=device)
         pipeline_options = PdfPipelineOptions(
+            enable_remote_services=enable_remote_services,
             accelerator_options=accelerator_options,
             do_ocr=ocr,
             ocr_options=ocr_options,

{docling-2.20.0 → docling-2.22.0}/docling/datamodel/base_models.py

@@ -39,6 +39,7 @@ class InputFormat(str, Enum):
     PDF = "pdf"
     ASCIIDOC = "asciidoc"
     MD = "md"
+    CSV = "csv"
     XLSX = "xlsx"
     XML_USPTO = "xml_uspto"
     JSON_DOCLING = "json_docling"
@@ -61,6 +62,7 @@ FormatToExtensions: Dict[InputFormat, List[str]] = {
     InputFormat.XML_PUBMED: ["xml", "nxml"],
     InputFormat.IMAGE: ["jpg", "jpeg", "png", "tif", "tiff", "bmp"],
     InputFormat.ASCIIDOC: ["adoc", "asciidoc", "asc"],
+    InputFormat.CSV: ["csv"],
     InputFormat.XLSX: ["xlsx"],
     InputFormat.XML_USPTO: ["xml", "txt"],
     InputFormat.JSON_DOCLING: ["json"],
@@ -88,6 +90,7 @@ FormatToMimeType: Dict[InputFormat, List[str]] = {
     InputFormat.PDF: ["application/pdf"],
     InputFormat.ASCIIDOC: ["text/asciidoc"],
     InputFormat.MD: ["text/markdown", "text/x-markdown"],
+    InputFormat.CSV: ["text/csv"],
     InputFormat.XLSX: [
         "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"
     ],

{docling-2.20.0 → docling-2.22.0}/docling/datamodel/document.py

@@ -1,3 +1,4 @@
+import csv
 import logging
 import re
 from enum import Enum
@@ -296,6 +297,7 @@ class _DocumentConversionInput(BaseModel):
                 mime = _DocumentConversionInput._mime_from_extension(ext)
 
         mime = mime or _DocumentConversionInput._detect_html_xhtml(content)
+        mime = mime or _DocumentConversionInput._detect_csv(content)
         mime = mime or "text/plain"
         formats = MimeTypeToFormat.get(mime, [])
         if formats:
@@ -352,6 +354,8 @@ class _DocumentConversionInput(BaseModel):
             mime = FormatToMimeType[InputFormat.HTML][0]
         elif ext in FormatToExtensions[InputFormat.MD]:
             mime = FormatToMimeType[InputFormat.MD][0]
+        elif ext in FormatToExtensions[InputFormat.CSV]:
+            mime = FormatToMimeType[InputFormat.CSV][0]
         elif ext in FormatToExtensions[InputFormat.JSON_DOCLING]:
             mime = FormatToMimeType[InputFormat.JSON_DOCLING][0]
         elif ext in FormatToExtensions[InputFormat.PDF]:
@@ -392,3 +396,32 @@ class _DocumentConversionInput(BaseModel):
             return "application/xml"
 
         return None
+
+    @staticmethod
+    def _detect_csv(
+        content: bytes,
+    ) -> Optional[Literal["text/csv"]]:
+        """Guess the mime type of a CSV file from its content.
+
+        Args:
+            content: A short piece of a document from its beginning.
+
+        Returns:
+            The mime type of a CSV file, or None if the content does
+              not match any of the format.
+        """
+        content_str = content.decode("ascii", errors="ignore").strip()
+
+        # Ensure there's at least one newline (CSV is usually multi-line)
+        if "\n" not in content_str:
+            return None
+
+        # Use csv.Sniffer to detect CSV characteristics
+        try:
+            dialect = csv.Sniffer().sniff(content_str)
+            if dialect.delimiter in {",", ";", "\t", "|"}:  # Common delimiters
+                return "text/csv"
+        except csv.Error:
+            return None
+
+        return None

{docling-2.20.0 → docling-2.22.0}/docling/datamodel/pipeline_options.py

@@ -257,6 +257,7 @@ class PipelineOptions(BaseModel):
     )
     document_timeout: Optional[float] = None
     accelerator_options: AcceleratorOptions = AcceleratorOptions()
+    enable_remote_services: bool = False
 
 
 class PdfPipelineOptions(PipelineOptions):

{docling-2.20.0 → docling-2.22.0}/docling/datamodel/settings.py

@@ -1,6 +1,6 @@
 import sys
 from pathlib import Path
-from typing import Annotated, Tuple
+from typing import Annotated, Optional, Tuple
 
 from pydantic import BaseModel, PlainValidator
 from pydantic_settings import BaseSettings, SettingsConfigDict
@@ -62,6 +62,7 @@ class AppSettings(BaseSettings):
     debug: DebugSettings
 
     cache_dir: Path = Path.home() / ".cache" / "docling"
+    artifacts_path: Optional[Path] = None
 
 
 settings = AppSettings(perf=BatchConcurrencySettings(), debug=DebugSettings())

{docling-2.20.0 → docling-2.22.0}/docling/document_converter.py

@@ -10,6 +10,7 @@ from pydantic import BaseModel, ConfigDict, model_validator, validate_call
 
 from docling.backend.abstract_backend import AbstractDocumentBackend
 from docling.backend.asciidoc_backend import AsciiDocBackend
+from docling.backend.csv_backend import CsvDocumentBackend
 from docling.backend.docling_parse_v2_backend import DoclingParseV2DocumentBackend
 from docling.backend.html_backend import HTMLDocumentBackend
 from docling.backend.json.docling_json_backend import DoclingJSONBackend
@@ -61,6 +62,11 @@ class FormatOption(BaseModel):
         return self
 
 
+class CsvFormatOption(FormatOption):
+    pipeline_cls: Type = SimplePipeline
+    backend: Type[AbstractDocumentBackend] = CsvDocumentBackend
+
+
 class ExcelFormatOption(FormatOption):
     pipeline_cls: Type = SimplePipeline
     backend: Type[AbstractDocumentBackend] = MsExcelDocumentBackend
@@ -113,6 +119,9 @@ class PdfFormatOption(FormatOption):
 
 def _get_default_option(format: InputFormat) -> FormatOption:
     format_to_default_options = {
+        InputFormat.CSV: FormatOption(
+            pipeline_cls=SimplePipeline, backend=CsvDocumentBackend
+        ),
         InputFormat.XLSX: FormatOption(
             pipeline_cls=SimplePipeline, backend=MsExcelDocumentBackend
         ),

{docling-2.20.0 → docling-2.22.0}/docling/exceptions.py

@@ -4,3 +4,7 @@ class BaseError(RuntimeError):
 
 class ConversionError(BaseError):
     pass
+
+
+class OperationNotAllowed(BaseError):
+    pass

{docling-2.20.0 → docling-2.22.0}/docling/models/ds_glm_model.py

@@ -4,7 +4,12 @@ from pathlib import Path
 from typing import List, Union
 
 from deepsearch_glm.andromeda_nlp import nlp_model
-from docling_core.types.doc import BoundingBox, CoordOrigin, DoclingDocument
+from docling_core.types.doc import (
+    BoundingBox,
+    CoordOrigin,
+    DocItemLabel,
+    DoclingDocument,
+)
 from docling_core.types.legacy_doc.base import BoundingBox as DsBoundingBox
 from docling_core.types.legacy_doc.base import (
     Figure,
@@ -71,12 +76,15 @@ class GlmModel:
         )
 
         main_text: List[Union[Ref, BaseText]] = []
+        page_headers: List[Union[Ref, BaseText]] = []
+        page_footers: List[Union[Ref, BaseText]] = []
+
         tables: List[DsSchemaTable] = []
         figures: List[Figure] = []
 
         page_no_to_page = {p.page_no: p for p in conv_res.pages}
 
-        for element in conv_res.assembled.elements:
+        for element in conv_res.assembled.body:
             # Convert bboxes to lower-left origin.
             target_bbox = DsBoundingBox(
                 element.cluster.bbox.to_bottom_left_origin(
@@ -238,6 +246,53 @@ class GlmModel:
                     )
                 )
 
+        # We can throw in headers and footers at the end of the legacy doc
+        # since the reading-order will re-sort it later.
+        for element in conv_res.assembled.headers:
+            # Convert bboxes to lower-left origin.
+            target_bbox = DsBoundingBox(
+                element.cluster.bbox.to_bottom_left_origin(
+                    page_no_to_page[element.page_no].size.height
+                ).as_tuple()
+            )
+
+            if isinstance(element, TextElement):
+
+                tel = BaseText(
+                    text=element.text,
+                    obj_type=layout_label_to_ds_type.get(element.label),
+                    name=element.label,
+                    prov=[
+                        Prov(
+                            bbox=target_bbox,
+                            page=element.page_no + 1,
+                            span=[0, len(element.text)],
+                        )
+                    ],
+                )
+                if element.label == DocItemLabel.PAGE_HEADER:
+                    index = len(page_headers)
+                    ref_str = f"#/page-headers/{index}"
+                    main_text.append(
+                        Ref(
+                            name=element.label,
+                            obj_type=layout_label_to_ds_type.get(element.label),
+                            ref=ref_str,
+                        ),
+                    )
+                    page_headers.append(tel)
+                elif element.label == DocItemLabel.PAGE_FOOTER:
+                    index = len(page_footers)
+                    ref_str = f"#/page-footers/{index}"
+                    main_text.append(
+                        Ref(
+                            name=element.label,
+                            obj_type=layout_label_to_ds_type.get(element.label),
+                            ref=ref_str,
+                        ),
+                    )
+                    page_footers.append(tel)
+
         page_dimensions = [
             PageDimensions(page=p.page_no + 1, height=p.size.height, width=p.size.width)
             for p in conv_res.pages
@@ -252,6 +307,8 @@ class GlmModel:
             tables=tables,
             figures=figures,
             page_dimensions=page_dimensions,
+            page_headers=page_headers,
+            page_footers=page_footers,
         )
 
         return ds_doc
@@ -264,6 +321,7 @@ class GlmModel:
             glm_doc = self.model.apply_on_doc(ds_doc_dict)
 
             docling_doc: DoclingDocument = to_docling_document(glm_doc)  # Experimental
+            1 == 1
 
         # DEBUG code:
         def draw_clusters_and_cells(ds_document, page_no, show: bool = False):

{docling-2.20.0 → docling-2.22.0}/docling/models/picture_description_api_model.py

@@ -8,6 +8,7 @@ from PIL import Image
 from pydantic import BaseModel, ConfigDict
 
 from docling.datamodel.pipeline_options import PictureDescriptionApiOptions
+from docling.exceptions import OperationNotAllowed
 from docling.models.picture_description_base_model import PictureDescriptionBaseModel
 
 _log = logging.getLogger(__name__)
@@ -45,14 +46,20 @@ class ApiResponse(BaseModel):
 class PictureDescriptionApiModel(PictureDescriptionBaseModel):
     # elements_batch_size = 4
 
-    def __init__(self, enabled: bool, options: PictureDescriptionApiOptions):
+    def __init__(
+        self,
+        enabled: bool,
+        enable_remote_services: bool,
+        options: PictureDescriptionApiOptions,
+    ):
         super().__init__(enabled=enabled, options=options)
         self.options: PictureDescriptionApiOptions
 
         if self.enabled:
-            if options.url.host != "localhost":
-                raise NotImplementedError(
-                    "The options try to connect to remote APIs which are not yet allowed."
+            if not enable_remote_services:
+                raise OperationNotAllowed(
+                    "Connections to remote services is only allowed when set explicitly. "
+                    "pipeline_options.enable_remote_services=True."
                 )
 
     def _annotate_images(self, images: Iterable[Image.Image]) -> Iterable[str]:

{docling-2.20.0 → docling-2.22.0}/docling/models/tesseract_ocr_model.py

@@ -22,6 +22,7 @@ class TesseractOcrModel(BaseOcrModel):
         self.scale = 3  # multiplier for 72 dpi == 216 dpi.
         self.reader = None
         self.osd_reader = None
+        self.script_readers: dict[str, tesserocr.PyTessBaseAPI] = {}
 
         if self.enabled:
             install_errmsg = (
@@ -57,8 +58,6 @@ class TesseractOcrModel(BaseOcrModel):
             _log.debug("Initializing TesserOCR: %s", tesseract_version)
             lang = "+".join(self.options.lang)
 
-            self.script_readers: dict[str, tesserocr.PyTessBaseAPI] = {}
-
             if any([l.startswith("script/") for l in self._tesserocr_languages]):
                 self.script_prefix = "script/"
             else:

{docling-2.20.0 → docling-2.22.0}/docling/pipeline/standard_pdf_pipeline.py

@@ -61,6 +61,14 @@ class StandardPdfPipeline(PaginatedPipeline):
         artifacts_path: Optional[Path] = None
         if pipeline_options.artifacts_path is not None:
             artifacts_path = Path(pipeline_options.artifacts_path).expanduser()
+        elif settings.artifacts_path is not None:
+            artifacts_path = Path(settings.artifacts_path).expanduser()
+
+        if artifacts_path is not None and not artifacts_path.is_dir():
+            raise RuntimeError(
+                f"The value of {artifacts_path=} is not valid. "
+                "When defined, it must point to a folder containing all models required by the pipeline."
+            )
 
         self.keep_images = (
             self.pipeline_options.generate_page_images
@@ -201,6 +209,7 @@ class StandardPdfPipeline(PaginatedPipeline):
         ):
             return PictureDescriptionApiModel(
                 enabled=self.pipeline_options.do_picture_description,
+                enable_remote_services=self.pipeline_options.enable_remote_services,
                 options=self.pipeline_options.picture_description_options,
             )
         elif isinstance(

{docling-2.20.0 → docling-2.22.0}/docling/utils/glm_utils.py

@@ -15,6 +15,7 @@ from docling_core.types.doc import (
     TableCell,
     TableData,
 )
+from docling_core.types.doc.document import ContentLayer
 
 
 def resolve_item(paths, obj):
@@ -311,6 +312,15 @@ def to_docling_document(doc_glm, update_name_label=False) -> DoclingDocument:
                 current_list = None
 
                 doc.add_text(label=DocItemLabel.FORMULA, text="", orig=text, prov=prov)
+            elif label in [DocItemLabel.PAGE_HEADER, DocItemLabel.PAGE_FOOTER]:
+                current_list = None
+
+                doc.add_text(
+                    label=DocItemLabel(name_label),
+                    text=text,
+                    prov=prov,
+                    content_layer=ContentLayer.FURNITURE,
+                )
             else:
                 current_list = None
 

{docling-2.20.0 → docling-2.22.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "docling"
-version = "2.20.0"  # DO NOT EDIT, updated automatically
+version = "2.22.0"  # DO NOT EDIT, updated automatically
 description = "SDK and CLI for parsing PDF, DOCX, HTML, and more, to a unified document representation for powering downstream workflows such as gen AI applications."
 authors = ["Christoph Auer <cau@zurich.ibm.com>", "Michele Dolfi <dol@zurich.ibm.com>", "Maxim Lysak <mly@zurich.ibm.com>", "Nikos Livathinos <nli@zurich.ibm.com>", "Ahmed Nassar <ahn@zurich.ibm.com>", "Panos Vagenas <pva@zurich.ibm.com>", "Peter Staar <taa@zurich.ibm.com>"]
 license = "MIT"
@@ -26,7 +26,7 @@ packages = [{include = "docling"}]
 ######################
 python = "^3.9"
 pydantic = "^2.0.0"
-docling-core = {extras = ["chunking"], version = "^2.17.2"}
+docling-core = {extras = ["chunking"], version = "^2.18.0"}
 docling-ibm-models = "^3.3.0"
 deepsearch-glm = "^1.0.0"
 docling-parse = "^3.3.0"
@@ -63,7 +63,7 @@ transformers = [
   {markers = "sys_platform != 'darwin' or platform_machine != 'x86_64'", version = "^4.46.0", optional = true },
   {markers = "sys_platform == 'darwin' and platform_machine == 'x86_64'", version = "~4.42.0", optional = true }
 ]
-pillow = "^10.0.0"
+pillow = ">=10.0.0,<12.0.0"
 tqdm = "^4.65.0"
 
 [tool.poetry.group.dev.dependencies]