docling 1.7.1__py3-none-any.whl → 1.8.0__py3-none-any.whl

docling/backend/abstract_backend.py

@@ -7,8 +7,6 @@ from PIL import Image
 
 
 class PdfPageBackend(ABC):
-    def __init__(self, page_obj: Any) -> object:
-        pass
 
     @abstractmethod
     def get_text_in_rect(self, bbox: "BoundingBox") -> str:
@@ -32,6 +30,10 @@ class PdfPageBackend(ABC):
     def get_size(self) -> "PageSize":
         pass
 
+    @abstractmethod
+    def is_valid(self) -> bool:
+        pass
+
     @abstractmethod
     def unload(self):
         pass

docling/backend/docling_parse_backend.py

@@ -19,22 +19,23 @@ class DoclingParsePageBackend(PdfPageBackend):
     def __init__(
         self, parser: pdf_parser, document_hash: str, page_no: int, page_obj: PdfPage
     ):
-        super().__init__(page_obj)
         self._ppage = page_obj
-
         parsed_page = parser.parse_pdf_from_key_on_page(document_hash, page_no)
 
         self._dpage = None
-        self.broken_page = "pages" not in parsed_page
-        if not self.broken_page:
+        self.valid = "pages" in parsed_page
+        if self.valid:
             self._dpage = parsed_page["pages"][0]
         else:
-            raise RuntimeError(
-                f"Page {page_no} of document {document_hash} could not be parsed."
+            _log.info(
+                f"An error occured when loading page {page_no} of document {document_hash}."
             )
 
+    def is_valid(self) -> bool:
+        return self.valid
+
     def get_text_in_rect(self, bbox: BoundingBox) -> str:
-        if self.broken_page:
+        if not self.valid:
             return ""
         # Find intersecting cells on the page
         text_piece = ""
@@ -70,7 +71,7 @@ class DoclingParsePageBackend(PdfPageBackend):
         cells = []
         cell_counter = 0
 
-        if self.broken_page:
+        if not self.valid:
             return cells
 
         page_size = self.get_size()
@@ -201,7 +202,9 @@ class DoclingParseDocumentBackend(PdfDocumentBackend):
             success = self.parser.load_document(document_hash, str(path_or_stream))
 
         if not success:
-            raise RuntimeError("docling-parse could not load this document.")
+            raise RuntimeError(
+                f"docling-parse could not load document {document_hash}."
+            )
 
     def page_count(self) -> int:
         return len(self._pdoc)  # To be replaced with docling-parse API

docling/backend/pypdfium2_backend.py

@@ -1,3 +1,4 @@
+import logging
 import random
 from io import BytesIO
 from pathlib import Path
@@ -7,17 +8,32 @@ import pypdfium2 as pdfium
 import pypdfium2.raw as pdfium_c
 from PIL import Image, ImageDraw
 from pypdfium2 import PdfPage
+from pypdfium2._helpers.misc import PdfiumError
 
 from docling.backend.abstract_backend import PdfDocumentBackend, PdfPageBackend
 from docling.datamodel.base_models import BoundingBox, Cell, CoordOrigin, PageSize
 
+_log = logging.getLogger(__name__)
+
 
 class PyPdfiumPageBackend(PdfPageBackend):
-    def __init__(self, page_obj: PdfPage):
-        super().__init__(page_obj)
-        self._ppage = page_obj
+    def __init__(
+        self, pdfium_doc: pdfium.PdfDocument, document_hash: str, page_no: int
+    ):
+        self.valid = True  # No better way to tell from pypdfium.
+        try:
+            self._ppage: pdfium.PdfPage = pdfium_doc[page_no]
+        except PdfiumError as e:
+            _log.info(
+                f"An exception occured when loading page {page_no} of document {document_hash}.",
+                exc_info=True,
+            )
+            self.valid = False
         self.text_page = None
 
+    def is_valid(self) -> bool:
+        return self.valid
+
     def get_bitmap_rects(self, scale: int = 1) -> Iterable[BoundingBox]:
         AREA_THRESHOLD = 32 * 32
         for obj in self._ppage.get_objects(filter=[pdfium_c.FPDF_PAGEOBJ_IMAGE]):
@@ -217,13 +233,18 @@ class PyPdfiumPageBackend(PdfPageBackend):
 class PyPdfiumDocumentBackend(PdfDocumentBackend):
     def __init__(self, path_or_stream: Union[BytesIO, Path], document_hash: str):
         super().__init__(path_or_stream, document_hash)
-        self._pdoc = pdfium.PdfDocument(path_or_stream)
+        try:
+            self._pdoc = pdfium.PdfDocument(path_or_stream)
+        except PdfiumError as e:
+            raise RuntimeError(
+                f"pypdfium could not load document {document_hash}"
+            ) from e
 
     def page_count(self) -> int:
         return len(self._pdoc)
 
     def load_page(self, page_no: int) -> PyPdfiumPageBackend:
-        return PyPdfiumPageBackend(self._pdoc[page_no])
+        return PyPdfiumPageBackend(self._pdoc, self.document_hash, page_no)
 
     def is_valid(self) -> bool:
         return self.page_count() > 0

docling/datamodel/base_models.py

@@ -16,7 +16,7 @@ class ConversionStatus(str, Enum):
     STARTED = auto()
     FAILURE = auto()
     SUCCESS = auto()
-    SUCCESS_WITH_ERRORS = auto()
+    PARTIAL_SUCCESS = auto()
 
 
 class DocInputType(str, Enum):
@@ -29,6 +29,18 @@ class CoordOrigin(str, Enum):
     BOTTOMLEFT = auto()
 
 
+class DoclingComponentType(str, Enum):
+    PDF_BACKEND = auto()
+    MODEL = auto()
+    DOC_ASSEMBLER = auto()
+
+
+class ErrorItem(BaseModel):
+    component_type: DoclingComponentType
+    module_name: str
+    error_message: str
+
+
 class PageSize(BaseModel):
     width: float = 0.0
     height: float = 0.0

docling/datamodel/document.py

@@ -19,6 +19,7 @@ from docling.datamodel.base_models import (
     AssembledUnit,
     ConversionStatus,
     DocumentStream,
+    ErrorItem,
     FigureElement,
     Page,
     PageElement,
@@ -118,7 +119,7 @@ class ConvertedDocument(BaseModel):
     input: InputDocument
 
     status: ConversionStatus = ConversionStatus.PENDING  # failure, success
-    errors: List[Dict] = []  # structure to keep errors
+    errors: List[ErrorItem] = []  # structure to keep errors
 
     pages: List[Page] = []
     assembled: Optional[AssembledUnit] = None

docling/document_converter.py

@@ -16,6 +16,8 @@ from docling.datamodel.base_models import (
     AssembledUnit,
     AssembleOptions,
     ConversionStatus,
+    DoclingComponentType,
+    ErrorItem,
     Page,
     PipelineOptions,
 )
@@ -157,7 +159,6 @@ class DocumentConverter:
             for page_batch in chunkify(
                 converted_doc.pages, settings.perf.page_batch_size
             ):
-
                 start_pb_time = time.time()
                 # Pipeline
 
@@ -205,12 +206,27 @@ class DocumentConverter:
             converted_doc.pages = all_assembled_pages
             self.assemble_doc(converted_doc)
 
-            converted_doc.status = ConversionStatus.SUCCESS
+            status = ConversionStatus.SUCCESS
+            for page in converted_doc.pages:
+                if not page._backend.is_valid():
+                    converted_doc.errors.append(
+                        ErrorItem(
+                            component_type=DoclingComponentType.PDF_BACKEND,
+                            module_name=type(page._backend).__name__,
+                            error_message=f"Page {page.page_no} failed to parse.",
+                        )
+                    )
+                    status = ConversionStatus.PARTIAL_SUCCESS
+
+            converted_doc.status = status
 
         except Exception as e:
             converted_doc.status = ConversionStatus.FAILURE
             trace = "\n".join(traceback.format_exception(e))
-            _log.info(f"Encountered an error during conversion: {trace}")
+            _log.info(
+                f"Encountered an error during conversion of document {in_doc.document_hash}:\n"
+                f"{trace}"
+            )
 
         end_doc_time = time.time() - start_doc_time
         _log.info(
@@ -230,7 +246,9 @@ class DocumentConverter:
     # Generate the page image and store it in the page object
     def populate_page_images(self, doc: InputDocument, page: Page) -> Page:
         # default scale
-        page.get_image(scale=1.0)
+        page.get_image(
+            scale=1.0
+        )  # puts the page image on the image cache at default scale
 
         # user requested scales
         if self.assemble_options.images_scale is not None:

{docling-1.7.1.dist-info → docling-1.8.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: docling
-Version: 1.7.1
+Version: 1.8.0
 Summary: Docling PDF conversion package
 Home-page: https://github.com/DS4SD/docling
 License: MIT

{docling-1.7.1.dist-info → docling-1.8.0.dist-info}/RECORD

@@ -1,13 +1,13 @@
 docling/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling/backend/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling/backend/abstract_backend.py,sha256=wvrywm1pPt79L5Dt_da5QGmf9dDzjBGj1rSaUQxqI8s,1432
-docling/backend/docling_parse_backend.py,sha256=hXyF2VPPdLs7APWEXTlfz0wI86rUGYa67Q73zgTB-Ug,7438
-docling/backend/pypdfium2_backend.py,sha256=84AnFah8Ztk-j8_9MTHalPU3a9fClrEz7A_rfzWDkFc,8122
+docling/backend/abstract_backend.py,sha256=xfNNiZKksPPa9KAiA-fHD86flg0It4n_29ccpm8fFiY,1436
+docling/backend/docling_parse_backend.py,sha256=r3aJwsWR7qG47ElhOa9iQJJQauHMt950FfCsf6fhlP4,7480
+docling/backend/pypdfium2_backend.py,sha256=FggVFitmyMMmLar6vk6XQsavGOPQx95TD14opWYRMAY,8837
 docling/datamodel/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-docling/datamodel/base_models.py,sha256=5VHit5h7OleKnbhvy-sWDxQLizEdNrGUBrypyzwHyAE,8604
-docling/datamodel/document.py,sha256=J97KeT8fJRKijUorDky-xA2FoOGBXOjrReYjdeo8NK4,13333
+docling/datamodel/base_models.py,sha256=DHpdLvEPpBaQsZ9gFmAKRiXdC2HwoApE9ufCw7GEhx4,8827
+docling/datamodel/document.py,sha256=0KT_G6VGbbLoT8BNiMjzNOWF-c_USQEOP0Bb_rgGMUo,13353
 docling/datamodel/settings.py,sha256=t5g6wrEJnPa9gBzMMl8ppgBRUYz-8xgopEtfMS0ZH28,733
-docling/document_converter.py,sha256=Tx4BwtOxpwizmXgJl2nK6I-6m0V3fygHwYFomJTH2Ns,10433
+docling/document_converter.py,sha256=pfHrJfvUKAAFNY8MeECwjvschrRQbV6qbDM72x7aw60,11187
 docling/models/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling/models/base_ocr_model.py,sha256=Ipl82a3AV2OsgMQSMEMpnWJ6MXcmyIQzmp52PmTaB0g,4465
 docling/models/ds_glm_model.py,sha256=wmb--2JKFQby-kvidw6PyM8wURPXYPQ_Z_eKKCBAdYQ,3192
@@ -21,7 +21,7 @@ docling/pipeline/standard_model_pipeline.py,sha256=UTjyaEXvz9htYZz-IMTkn11cZwNjg
 docling/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 docling/utils/layout_utils.py,sha256=FOFbL0hKzUoWXdZaeUvEtFqKv0IkPifIr4sdGW4suKs,31804
 docling/utils/utils.py,sha256=llhXSbIDNZ1MHOwBEfLHBAoJIAYI7QlPIonlI1jLUJ0,1208
-docling-1.7.1.dist-info/LICENSE,sha256=ACwmltkrXIz5VsEQcrqljq-fat6ZXAMepjXGoe40KtE,1069
-docling-1.7.1.dist-info/METADATA,sha256=ADdVabYgc4VEIGKhM-tI6XBU_CG9tzKl_au69TZ9LbY,7229
-docling-1.7.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-docling-1.7.1.dist-info/RECORD,,
+docling-1.8.0.dist-info/LICENSE,sha256=ACwmltkrXIz5VsEQcrqljq-fat6ZXAMepjXGoe40KtE,1069
+docling-1.8.0.dist-info/METADATA,sha256=Z9S-zY0YuOT_xPHbbjWzdOCNQcnEiEpaZ7jSApZpUF0,7229
+docling-1.8.0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+docling-1.8.0.dist-info/RECORD,,