docling 0.1.2__tar.gz

docling-0.1.2/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) [year] [fullname]
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

docling-0.1.2/PKG-INFO

@@ -0,0 +1,132 @@
+Metadata-Version: 2.1
+Name: docling
+Version: 0.1.2
+Summary: Docling PDF conversion package
+Home-page: https://github.com/DS4SD/docling
+License: MIT
+Keywords: docling,convert,document,pdf,layout model,segmentation,table structure,table former
+Author: Christoph Auer
+Author-email: cau@zurich.ibm.com
+Requires-Python: >=3.11,<4.0
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: MacOS :: MacOS X
+Classifier: Operating System :: POSIX :: Linux
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Requires-Dist: deepsearch-glm (>=0.18.4,<1)
+Requires-Dist: deepsearch-toolkit (>=0.47.0,<1)
+Requires-Dist: docling-core (>=0.2.0,<0.3.0)
+Requires-Dist: docling-ibm-models (>=0.2.0,<0.3.0)
+Requires-Dist: filetype (>=1.2.0,<2.0.0)
+Requires-Dist: huggingface_hub (>=0.23,<1)
+Requires-Dist: pydantic (>=2.0.0,<3.0.0)
+Requires-Dist: pydantic-settings (>=2.3.0,<3.0.0)
+Requires-Dist: pypdfium2 (>=4.30.0,<5.0.0)
+Project-URL: Repository, https://github.com/DS4SD/docling
+Description-Content-Type: text/markdown
+
+<p align="center">
+  <a href="https://github.com/ds4sd/docling"> <img loading="lazy" alt="Docling" src="https://github.com/DS4SD/docling/raw/main/logo.png" width="150" /> </a>
+</p>
+
+# Docling
+
+Docling bundles PDF document conversion to JSON and Markdown in an easy, self-contained package.
+
+## Features
+* ⚡ Converts any PDF document to JSON or Markdown format, stable and lightning fast
+* 📑 Understands detailed page layout, reading order and recovers table structures
+* 📝 Extracts metadata from the document, such as title, authors, references and language
+* 🔍 Optionally applies OCR (use with scanned PDFs)
+
+## Setup
+
+You need Python 3.11 and poetry. Install poetry from [here](https://python-poetry.org/docs/#installing-with-the-official-installer).
+
+Once you have `poetry` installed, create an environment and install the package:
+
+```bash
+poetry env use $(which python3.11)
+poetry shell
+poetry install
+```
+
+**Notes**:
+* Works on macOS and Linux environments. Windows platforms are currently not tested.
+
+
+## Usage
+
+For basic usage, see the [convert.py](https://github.com/DS4SD/docling/blob/main/examples/convert.py) example module. Run with:
+
+```
+python examples/convert.py
+```
+The output of the above command will be written to `./scratch`.
+
+### Enable or disable pipeline features
+
+You can control if table structure recognition or OCR should be performed by arguments passed to `DocumentConverter` 
+```python
+doc_converter = DocumentConverter(
+    artifacts_path=artifacts_path,
+    pipeline_options=PipelineOptions(do_table_structure=False, # Controls if table structure is recovered. 
+                                     do_ocr=True), # Controls if OCR is applied (ignores programmatic content)
+)
+```
+
+### Impose limits on the document size
+
+You can limit the file size and number of pages which should be allowed to process per document.
+```python
+paths = [Path("./test/data/2206.01062.pdf")]
+
+input = DocumentConversionInput.from_paths(
+    paths, limits=DocumentLimits(max_num_pages=100, max_file_size=20971520)
+)
+```
+
+### Convert from binary PDF streams 
+
+You can convert PDFs from a binary stream instead of from the filesystem as follows:
+```python
+buf = BytesIO(your_binary_stream)
+docs = [DocumentStream(filename="my_doc.pdf", stream=buf)]
+input = DocumentConversionInput.from_streams(docs)
+converted_docs = doc_converter.convert(input)
+```
+### Limit resource usage
+
+You can limit the CPU threads used by `docling` by setting the environment variable `OMP_NUM_THREADS` accordingly. The default setting is using 4 CPU threads.
+
+
+## Contributing
+
+Please read [Contributing to Docling](https://github.com/DS4SD/docling/blob/main/CONTRIBUTING.md) for details.
+
+
+## References
+
+If you use `Docling` in your projects, please consider citing the following:
+
+```bib
+@software{Docling,
+author = {Deep Search Team},
+month = {7},
+title = {{Docling}},
+url = {https://github.com/DS4SD/docling},
+version = {main},
+year = {2024}
+}
+```
+
+## License
+
+The `Docling` codebase is under MIT license.
+For individual model usage, please refer to the model licenses found in the original packages.
+

docling-0.1.2/README.md

@@ -0,0 +1,99 @@
+<p align="center">
+  <a href="https://github.com/ds4sd/docling"> <img loading="lazy" alt="Docling" src="https://github.com/DS4SD/docling/raw/main/logo.png" width="150" /> </a>
+</p>
+
+# Docling
+
+Docling bundles PDF document conversion to JSON and Markdown in an easy, self-contained package.
+
+## Features
+* ⚡ Converts any PDF document to JSON or Markdown format, stable and lightning fast
+* 📑 Understands detailed page layout, reading order and recovers table structures
+* 📝 Extracts metadata from the document, such as title, authors, references and language
+* 🔍 Optionally applies OCR (use with scanned PDFs)
+
+## Setup
+
+You need Python 3.11 and poetry. Install poetry from [here](https://python-poetry.org/docs/#installing-with-the-official-installer).
+
+Once you have `poetry` installed, create an environment and install the package:
+
+```bash
+poetry env use $(which python3.11)
+poetry shell
+poetry install
+```
+
+**Notes**:
+* Works on macOS and Linux environments. Windows platforms are currently not tested.
+
+
+## Usage
+
+For basic usage, see the [convert.py](https://github.com/DS4SD/docling/blob/main/examples/convert.py) example module. Run with:
+
+```
+python examples/convert.py
+```
+The output of the above command will be written to `./scratch`.
+
+### Enable or disable pipeline features
+
+You can control if table structure recognition or OCR should be performed by arguments passed to `DocumentConverter` 
+```python
+doc_converter = DocumentConverter(
+    artifacts_path=artifacts_path,
+    pipeline_options=PipelineOptions(do_table_structure=False, # Controls if table structure is recovered. 
+                                     do_ocr=True), # Controls if OCR is applied (ignores programmatic content)
+)
+```
+
+### Impose limits on the document size
+
+You can limit the file size and number of pages which should be allowed to process per document.
+```python
+paths = [Path("./test/data/2206.01062.pdf")]
+
+input = DocumentConversionInput.from_paths(
+    paths, limits=DocumentLimits(max_num_pages=100, max_file_size=20971520)
+)
+```
+
+### Convert from binary PDF streams 
+
+You can convert PDFs from a binary stream instead of from the filesystem as follows:
+```python
+buf = BytesIO(your_binary_stream)
+docs = [DocumentStream(filename="my_doc.pdf", stream=buf)]
+input = DocumentConversionInput.from_streams(docs)
+converted_docs = doc_converter.convert(input)
+```
+### Limit resource usage
+
+You can limit the CPU threads used by `docling` by setting the environment variable `OMP_NUM_THREADS` accordingly. The default setting is using 4 CPU threads.
+
+
+## Contributing
+
+Please read [Contributing to Docling](https://github.com/DS4SD/docling/blob/main/CONTRIBUTING.md) for details.
+
+
+## References
+
+If you use `Docling` in your projects, please consider citing the following:
+
+```bib
+@software{Docling,
+author = {Deep Search Team},
+month = {7},
+title = {{Docling}},
+url = {https://github.com/DS4SD/docling},
+version = {main},
+year = {2024}
+}
+```
+
+## License
+
+The `Docling` codebase is under MIT license.
+For individual model usage, please refer to the model licenses found in the original packages.

docling-0.1.2/docling/backend/abstract_backend.py

@@ -0,0 +1,55 @@
+from abc import ABC, abstractmethod
+from io import BytesIO
+from pathlib import Path
+from typing import Any, Iterable, Optional, Union
+
+from PIL import Image
+
+
+class PdfPageBackend(ABC):
+    def __init__(self, page_obj: Any) -> object:
+        pass
+
+    @abstractmethod
+    def get_text_in_rect(self, bbox: "BoundingBox") -> str:
+        pass
+
+    @abstractmethod
+    def get_text_cells(self) -> Iterable["Cell"]:
+        pass
+
+    @abstractmethod
+    def get_page_image(
+        self, scale: int = 1, cropbox: Optional["BoundingBox"] = None
+    ) -> Image.Image:
+        pass
+
+    @abstractmethod
+    def get_size(self) -> "PageSize":
+        pass
+
+    @abstractmethod
+    def unload(self):
+        pass
+
+
+class PdfDocumentBackend(ABC):
+    @abstractmethod
+    def __init__(self, path_or_stream: Iterable[Union[BytesIO, Path]]):
+        pass
+
+    @abstractmethod
+    def load_page(self, page_no: int) -> PdfPageBackend:
+        pass
+
+    @abstractmethod
+    def page_count(self) -> int:
+        pass
+
+    @abstractmethod
+    def is_valid(self) -> bool:
+        pass
+
+    @abstractmethod
+    def unload(self):
+        pass

docling-0.1.2/docling/backend/pypdfium2_backend.py

@@ -0,0 +1,223 @@
+import random
+from io import BytesIO
+from pathlib import Path
+from typing import Iterable, List, Optional, Union
+
+import pypdfium2 as pdfium
+from PIL import Image, ImageDraw
+from pypdfium2 import PdfPage
+
+from docling.backend.abstract_backend import PdfDocumentBackend, PdfPageBackend
+from docling.datamodel.base_models import BoundingBox, Cell, CoordOrigin, PageSize
+
+
+class PyPdfiumPageBackend(PdfPageBackend):
+    def __init__(self, page_obj: PdfPage):
+        super().__init__(page_obj)
+        self._ppage = page_obj
+        self.text_page = None
+
+    def get_text_in_rect(self, bbox: BoundingBox) -> str:
+        if not self.text_page:
+            self.text_page = self._ppage.get_textpage()
+
+        if bbox.coord_origin != CoordOrigin.BOTTOMLEFT:
+            bbox = bbox.to_bottom_left_origin(self.get_size().height)
+
+        text_piece = self.text_page.get_text_bounded(*bbox.as_tuple())
+
+        return text_piece
+
+    def get_text_cells(self) -> Iterable[Cell]:
+        if not self.text_page:
+            self.text_page = self._ppage.get_textpage()
+
+        cells = []
+        cell_counter = 0
+
+        page_size = self.get_size()
+
+        for i in range(self.text_page.count_rects()):
+            rect = self.text_page.get_rect(i)
+            text_piece = self.text_page.get_text_bounded(*rect)
+            x0, y0, x1, y1 = rect
+            cells.append(
+                Cell(
+                    id=cell_counter,
+                    text=text_piece,
+                    bbox=BoundingBox(
+                        l=x0, b=y0, r=x1, t=y1, coord_origin=CoordOrigin.BOTTOMLEFT
+                    ).to_top_left_origin(page_size.height),
+                )
+            )
+            cell_counter += 1
+
+        # PyPdfium2 produces very fragmented cells, with sub-word level boundaries, in many PDFs.
+        # The cell merging code below is to clean this up.
+        def merge_horizontal_cells(
+            cells: List[Cell],
+            horizontal_threshold_factor: float = 1.0,
+            vertical_threshold_factor: float = 0.5,
+        ) -> List[Cell]:
+            if not cells:
+                return []
+
+            def group_rows(cells: List[Cell]) -> List[List[Cell]]:
+                rows = []
+                current_row = [cells[0]]
+                row_top = cells[0].bbox.t
+                row_bottom = cells[0].bbox.b
+                row_height = cells[0].bbox.height
+
+                for cell in cells[1:]:
+                    vertical_threshold = row_height * vertical_threshold_factor
+                    if (
+                        abs(cell.bbox.t - row_top) <= vertical_threshold
+                        and abs(cell.bbox.b - row_bottom) <= vertical_threshold
+                    ):
+                        current_row.append(cell)
+                        row_top = min(row_top, cell.bbox.t)
+                        row_bottom = max(row_bottom, cell.bbox.b)
+                        row_height = row_bottom - row_top
+                    else:
+                        rows.append(current_row)
+                        current_row = [cell]
+                        row_top = cell.bbox.t
+                        row_bottom = cell.bbox.b
+                        row_height = cell.bbox.height
+
+                if current_row:
+                    rows.append(current_row)
+
+                return rows
+
+            def merge_row(row: List[Cell]) -> List[Cell]:
+                merged = []
+                current_group = [row[0]]
+
+                for cell in row[1:]:
+                    prev_cell = current_group[-1]
+                    avg_height = (prev_cell.bbox.height + cell.bbox.height) / 2
+                    if (
+                        cell.bbox.l - prev_cell.bbox.r
+                        <= avg_height * horizontal_threshold_factor
+                    ):
+                        current_group.append(cell)
+                    else:
+                        merged.append(merge_group(current_group))
+                        current_group = [cell]
+
+                if current_group:
+                    merged.append(merge_group(current_group))
+
+                return merged
+
+            def merge_group(group: List[Cell]) -> Cell:
+                if len(group) == 1:
+                    return group[0]
+
+                merged_text = "".join(cell.text for cell in group)
+                merged_bbox = BoundingBox(
+                    l=min(cell.bbox.l for cell in group),
+                    t=min(cell.bbox.t for cell in group),
+                    r=max(cell.bbox.r for cell in group),
+                    b=max(cell.bbox.b for cell in group),
+                )
+                return Cell(id=group[0].id, text=merged_text, bbox=merged_bbox)
+
+            rows = group_rows(cells)
+            merged_cells = [cell for row in rows for cell in merge_row(row)]
+
+            for i, cell in enumerate(merged_cells, 1):
+                cell.id = i
+
+            return merged_cells
+
+        def draw_clusters_and_cells():
+            image = self.get_page_image()
+            draw = ImageDraw.Draw(image)
+            for c in cells:
+                x0, y0, x1, y1 = c.bbox.as_tuple()
+                cell_color = (
+                    random.randint(30, 140),
+                    random.randint(30, 140),
+                    random.randint(30, 140),
+                )
+                draw.rectangle([(x0, y0), (x1, y1)], outline=cell_color)
+            image.show()
+
+        # before merge:
+        # draw_clusters_and_cells()
+
+        cells = merge_horizontal_cells(cells)
+
+        # after merge:
+        # draw_clusters_and_cells()
+
+        return cells
+
+    def get_page_image(
+        self, scale: int = 1, cropbox: Optional[BoundingBox] = None
+    ) -> Image.Image:
+
+        page_size = self.get_size()
+
+        if not cropbox:
+            cropbox = BoundingBox(
+                l=0,
+                r=page_size.width,
+                t=0,
+                b=page_size.height,
+                coord_origin=CoordOrigin.TOPLEFT,
+            )
+            padbox = BoundingBox(
+                l=0, r=0, t=0, b=0, coord_origin=CoordOrigin.BOTTOMLEFT
+            )
+        else:
+            padbox = cropbox.to_bottom_left_origin(page_size.height)
+            padbox.r = page_size.width - padbox.r
+            padbox.t = page_size.height - padbox.t
+
+        image = (
+            self._ppage.render(
+                scale=scale * 1.5,
+                rotation=0,  # no additional rotation
+                crop=padbox.as_tuple(),
+            )
+            .to_pil()
+            .resize(size=(round(cropbox.width * scale), round(cropbox.height * scale)))
+        )  # We resize the image from 1.5x the given scale to make it sharper.
+
+        return image
+
+    def get_size(self) -> PageSize:
+        return PageSize(width=self._ppage.get_width(), height=self._ppage.get_height())
+
+    def unload(self):
+        self._ppage = None
+        self.text_page = None
+
+
+class PyPdfiumDocumentBackend(PdfDocumentBackend):
+    def __init__(self, path_or_stream: Iterable[Union[BytesIO, Path]]):
+        super().__init__(path_or_stream)
+
+        if isinstance(path_or_stream, Path):
+            self._pdoc = pdfium.PdfDocument(path_or_stream)
+        elif isinstance(path_or_stream, BytesIO):
+            self._pdoc = pdfium.PdfDocument(
+                path_or_stream
+            )  # TODO Fix me, won't accept bytes.
+
+    def page_count(self) -> int:
+        return len(self._pdoc)
+
+    def load_page(self, page_no: int) -> PdfPage:
+        return PyPdfiumPageBackend(self._pdoc[page_no])
+
+    def is_valid(self) -> bool:
+        return self.page_count() > 0
+
+    def unload(self):
+        self._pdoc.close()
+        self._pdoc = None