docling 2.48.0__py3-none-any.whl → 2.50.0__py3-none-any.whl

docling/models/rapid_ocr_model.py

@@ -79,6 +79,7 @@ class RapidOcrModel(BaseOcrModel):
                     "Cls.intra_op_num_threads": intra_op_num_threads,
                     # Recognition model settings
                     "Rec.model_path": self.options.rec_model_path,
+                    "Rec.font_path": self.options.rec_font_path,
                     "Rec.keys_path": self.options.rec_keys_path,
                     "Rec.use_cuda": use_cuda,
                     "Rec.use_dml": use_dml,

docling/models/table_structure_model.py

@@ -94,7 +94,7 @@ class TableStructureModel(BasePageModel):
     ) -> Path:
         return download_hf_model(
             repo_id="ds4sd/docling-models",
-            revision="v2.2.0",
+            revision="v2.3.0",
             local_dir=local_dir,
             force=force,
             progress=progress,

docling/models/vlm_models_inline/nuextract_transformers_model.py

@@ -0,0 +1,290 @@
+import logging
+import time
+from collections.abc import Iterable
+from pathlib import Path
+from typing import Any, Optional, Union
+
+import numpy as np
+from PIL.Image import Image
+from transformers import AutoModelForImageTextToText, AutoProcessor, GenerationConfig
+
+from docling.datamodel.accelerator_options import (
+    AcceleratorOptions,
+)
+from docling.datamodel.base_models import VlmPrediction
+from docling.datamodel.pipeline_options_vlm_model import InlineVlmOptions
+from docling.models.base_model import BaseVlmModel
+from docling.models.utils.hf_model_download import (
+    HuggingFaceModelDownloadMixin,
+)
+from docling.utils.accelerator_utils import decide_device
+
+_log = logging.getLogger(__name__)
+
+
+# Source code from https://huggingface.co/numind/NuExtract-2.0-8B
+def process_all_vision_info(messages, examples=None):
+    """
+    Process vision information from both messages and in-context examples, supporting batch processing.
+
+    Args:
+        messages: List of message dictionaries (single input) OR list of message lists (batch input)
+        examples: Optional list of example dictionaries (single input) OR list of example lists (batch)
+
+    Returns:
+        A flat list of all images in the correct order:
+        - For single input: example images followed by message images
+        - For batch input: interleaved as (item1 examples, item1 input, item2 examples, item2 input, etc.)
+        - Returns None if no images were found
+    """
+    try:
+        from qwen_vl_utils import fetch_image, process_vision_info
+    except ImportError:
+        raise ImportError(
+            "qwen-vl-utils is required for NuExtractTransformersModel. "
+            "Please install it with: pip install qwen-vl-utils"
+        )
+
+    from qwen_vl_utils import fetch_image, process_vision_info
+
+    # Helper function to extract images from examples
+    def extract_example_images(example_item):
+        if not example_item:
+            return []
+
+        # Handle both list of examples and single example
+        examples_to_process = (
+            example_item if isinstance(example_item, list) else [example_item]
+        )
+        images = []
+
+        for example in examples_to_process:
+            if (
+                isinstance(example.get("input"), dict)
+                and example["input"].get("type") == "image"
+            ):
+                images.append(fetch_image(example["input"]))
+
+        return images
+
+    # Normalize inputs to always be batched format
+    is_batch = messages and isinstance(messages[0], list)
+    messages_batch = messages if is_batch else [messages]
+    is_batch_examples = (
+        examples
+        and isinstance(examples, list)
+        and (isinstance(examples[0], list) or examples[0] is None)
+    )
+    examples_batch = (
+        examples
+        if is_batch_examples
+        else ([examples] if examples is not None else None)
+    )
+
+    # Ensure examples batch matches messages batch if provided
+    if examples and len(examples_batch) != len(messages_batch):
+        if not is_batch and len(examples_batch) == 1:
+            # Single example set for a single input is fine
+            pass
+        else:
+            raise ValueError("Examples batch length must match messages batch length")
+
+    # Process all inputs, maintaining correct order
+    all_images = []
+    for i, message_group in enumerate(messages_batch):
+        # Get example images for this input
+        if examples and i < len(examples_batch):
+            input_example_images = extract_example_images(examples_batch[i])
+            all_images.extend(input_example_images)
+
+        # Get message images for this input
+        input_message_images = process_vision_info(message_group)[0] or []
+        all_images.extend(input_message_images)
+
+    return all_images if all_images else None
+
+
+class NuExtractTransformersModel(BaseVlmModel, HuggingFaceModelDownloadMixin):
+    def __init__(
+        self,
+        enabled: bool,
+        artifacts_path: Optional[Path],
+        accelerator_options: AcceleratorOptions,
+        vlm_options: InlineVlmOptions,
+    ):
+        self.enabled = enabled
+        self.vlm_options = vlm_options
+
+        if self.enabled:
+            import torch
+
+            self.device = decide_device(
+                accelerator_options.device,
+                supported_devices=vlm_options.supported_devices,
+            )
+            _log.debug(f"Available device for NuExtract VLM: {self.device}")
+
+            self.max_new_tokens = vlm_options.max_new_tokens
+            self.temperature = vlm_options.temperature
+
+            repo_cache_folder = vlm_options.repo_id.replace("/", "--")
+
+            if artifacts_path is None:
+                artifacts_path = self.download_models(self.vlm_options.repo_id)
+            elif (artifacts_path / repo_cache_folder).exists():
+                artifacts_path = artifacts_path / repo_cache_folder
+
+            self.processor = AutoProcessor.from_pretrained(
+                artifacts_path,
+                trust_remote_code=vlm_options.trust_remote_code,
+                use_fast=True,
+            )
+            self.processor.tokenizer.padding_side = "left"
+
+            self.vlm_model = AutoModelForImageTextToText.from_pretrained(
+                artifacts_path,
+                device_map=self.device,
+                torch_dtype=self.vlm_options.torch_dtype,
+                _attn_implementation=(
+                    "flash_attention_2"
+                    if self.device.startswith("cuda")
+                    and accelerator_options.cuda_use_flash_attention2
+                    else "sdpa"
+                ),
+                trust_remote_code=vlm_options.trust_remote_code,
+            )
+            self.vlm_model = torch.compile(self.vlm_model)  # type: ignore
+
+            # Load generation config
+            self.generation_config = GenerationConfig.from_pretrained(artifacts_path)
+
+    def process_images(
+        self,
+        image_batch: Iterable[Union[Image, np.ndarray]],
+        prompt: Union[str, list[str]],
+    ) -> Iterable[VlmPrediction]:
+        """
+        Batched inference for NuExtract VLM using the specialized input format.
+
+        Args:
+            image_batch: Iterable of PIL Images or numpy arrays
+            prompt: Either:
+                - str: Single template used for all images
+                - list[str]: List of templates (one per image, must match image count)
+        """
+        import torch
+        from PIL import Image as PILImage
+
+        # Normalize images to RGB PIL
+        pil_images: list[Image] = []
+        for img in image_batch:
+            if isinstance(img, np.ndarray):
+                if img.ndim == 3 and img.shape[2] in (3, 4):
+                    pil_img = PILImage.fromarray(img.astype(np.uint8))
+                elif img.ndim == 2:
+                    pil_img = PILImage.fromarray(img.astype(np.uint8), mode="L")
+                else:
+                    raise ValueError(f"Unsupported numpy array shape: {img.shape}")
+            else:
+                pil_img = img
+            if pil_img.mode != "RGB":
+                pil_img = pil_img.convert("RGB")
+            pil_images.append(pil_img)
+
+        if not pil_images:
+            return
+
+        # Normalize templates (1 per image)
+        if isinstance(prompt, str):
+            templates = [prompt] * len(pil_images)
+        else:
+            if len(prompt) != len(pil_images):
+                raise ValueError(
+                    f"Number of templates ({len(prompt)}) must match number of images ({len(pil_images)})"
+                )
+            templates = prompt
+
+        # Construct NuExtract input format
+        inputs = []
+        for pil_img, template in zip(pil_images, templates):
+            input_item = {
+                "document": {"type": "image", "image": pil_img},
+                "template": template,
+            }
+            inputs.append(input_item)
+
+        # Create messages structure for batch processing
+        messages = [
+            [
+                {
+                    "role": "user",
+                    "content": [x["document"]],
+                }
+            ]
+            for x in inputs
+        ]
+
+        # Apply chat template to each example individually
+        texts = [
+            self.processor.tokenizer.apply_chat_template(
+                messages[i],
+                template=x["template"],
+                tokenize=False,
+                add_generation_prompt=True,
+            )
+            for i, x in enumerate(inputs)
+        ]
+
+        # Process vision inputs using qwen-vl-utils
+        image_inputs = process_all_vision_info(messages)
+
+        # Process with the processor
+        processor_inputs = self.processor(
+            text=texts,
+            images=image_inputs,
+            padding=True,
+            return_tensors="pt",
+            **self.vlm_options.extra_processor_kwargs,
+        )
+        processor_inputs = {k: v.to(self.device) for k, v in processor_inputs.items()}
+
+        # Generate
+        gen_kwargs = {
+            **processor_inputs,
+            "max_new_tokens": self.max_new_tokens,
+            "generation_config": self.generation_config,
+            **self.vlm_options.extra_generation_config,
+        }
+        if self.temperature > 0:
+            gen_kwargs["do_sample"] = True
+            gen_kwargs["temperature"] = self.temperature
+        else:
+            gen_kwargs["do_sample"] = False
+
+        start_time = time.time()
+        with torch.inference_mode():
+            generated_ids = self.vlm_model.generate(**gen_kwargs)
+        generation_time = time.time() - start_time
+
+        # Trim generated sequences
+        input_len = processor_inputs["input_ids"].shape[1]
+        trimmed_sequences = generated_ids[:, input_len:]
+
+        # Decode with the processor/tokenizer
+        decoded_texts: list[str] = self.processor.batch_decode(
+            trimmed_sequences,
+            skip_special_tokens=True,
+            clean_up_tokenization_spaces=False,
+        )
+
+        # Optional logging
+        if generated_ids.shape[0] > 0:  # type: ignore
+            _log.debug(
+                f"Generated {int(generated_ids[0].shape[0])} tokens in {generation_time:.2f}s "
+                f"for batch size {generated_ids.shape[0]}."  # type: ignore
+            )
+
+        for text in decoded_texts:
+            # Apply decode_response to the output text
+            decoded_text = self.vlm_options.decode_response(text)
+            yield VlmPrediction(text=decoded_text, generation_time=generation_time)

docling/pipeline/base_extraction_pipeline.py

@@ -0,0 +1,58 @@
+import logging
+from abc import ABC, abstractmethod
+from typing import Optional
+
+from docling.datamodel.base_models import ConversionStatus, ErrorItem
+from docling.datamodel.document import InputDocument
+from docling.datamodel.extraction import ExtractionResult, ExtractionTemplateType
+from docling.datamodel.pipeline_options import BaseOptions
+
+_log = logging.getLogger(__name__)
+
+
+class BaseExtractionPipeline(ABC):
+    def __init__(self, pipeline_options: BaseOptions):
+        self.pipeline_options = pipeline_options
+
+    def execute(
+        self,
+        in_doc: InputDocument,
+        raises_on_error: bool,
+        template: Optional[ExtractionTemplateType] = None,
+    ) -> ExtractionResult:
+        ext_res = ExtractionResult(input=in_doc)
+
+        try:
+            ext_res = self._extract_data(ext_res, template)
+            ext_res.status = self._determine_status(ext_res)
+        except Exception as e:
+            ext_res.status = ConversionStatus.FAILURE
+            error_item = ErrorItem(
+                component_type="extraction_pipeline",
+                module_name=self.__class__.__name__,
+                error_message=str(e),
+            )
+            ext_res.errors.append(error_item)
+            if raises_on_error:
+                raise e
+
+        return ext_res
+
+    @abstractmethod
+    def _extract_data(
+        self,
+        ext_res: ExtractionResult,
+        template: Optional[ExtractionTemplateType] = None,
+    ) -> ExtractionResult:
+        """Subclass must populate ext_res.pages/errors and return the result."""
+        raise NotImplementedError
+
+    @abstractmethod
+    def _determine_status(self, ext_res: ExtractionResult) -> ConversionStatus:
+        """Subclass must decide SUCCESS/PARTIAL_SUCCESS/FAILURE based on ext_res."""
+        raise NotImplementedError
+
+    @classmethod
+    @abstractmethod
+    def get_default_options(cls) -> BaseOptions:
+        pass

docling/pipeline/extraction_vlm_pipeline.py

@@ -0,0 +1,204 @@
+import inspect
+import json
+import logging
+from pathlib import Path
+from typing import Optional
+
+from PIL.Image import Image
+from pydantic import BaseModel
+
+from docling.backend.abstract_backend import PaginatedDocumentBackend
+from docling.backend.pdf_backend import PdfDocumentBackend
+from docling.datamodel.base_models import ConversionStatus, ErrorItem
+from docling.datamodel.document import InputDocument
+from docling.datamodel.extraction import (
+    ExtractedPageData,
+    ExtractionResult,
+    ExtractionTemplateType,
+)
+from docling.datamodel.pipeline_options import BaseOptions, VlmExtractionPipelineOptions
+from docling.datamodel.settings import settings
+from docling.models.vlm_models_inline.nuextract_transformers_model import (
+    NuExtractTransformersModel,
+)
+from docling.pipeline.base_extraction_pipeline import BaseExtractionPipeline
+from docling.utils.accelerator_utils import decide_device
+
+_log = logging.getLogger(__name__)
+
+
+class ExtractionVlmPipeline(BaseExtractionPipeline):
+    def __init__(self, pipeline_options: VlmExtractionPipelineOptions):
+        super().__init__(pipeline_options)
+
+        # Initialize VLM model with default options
+        self.accelerator_options = pipeline_options.accelerator_options
+        self.pipeline_options: VlmExtractionPipelineOptions
+
+        artifacts_path: Optional[Path] = None
+        if pipeline_options.artifacts_path is not None:
+            artifacts_path = Path(pipeline_options.artifacts_path).expanduser()
+        elif settings.artifacts_path is not None:
+            artifacts_path = Path(settings.artifacts_path).expanduser()
+
+        if artifacts_path is not None and not artifacts_path.is_dir():
+            raise RuntimeError(
+                f"The value of {artifacts_path=} is not valid. "
+                "When defined, it must point to a folder containing all models required by the pipeline."
+            )
+
+        # Create VLM model instance
+        self.vlm_model = NuExtractTransformersModel(
+            enabled=True,
+            artifacts_path=artifacts_path,  # Will download automatically
+            accelerator_options=self.accelerator_options,
+            vlm_options=pipeline_options.vlm_options,
+        )
+
+    def _extract_data(
+        self,
+        ext_res: ExtractionResult,
+        template: Optional[ExtractionTemplateType] = None,
+    ) -> ExtractionResult:
+        """Extract data using the VLM model."""
+        try:
+            # Get images from input document using the backend
+            images = self._get_images_from_input(ext_res.input)
+            if not images:
+                ext_res.status = ConversionStatus.FAILURE
+                ext_res.errors.append(
+                    ErrorItem(
+                        component_type="extraction_pipeline",
+                        module_name=self.__class__.__name__,
+                        error_message="No images found in document",
+                    )
+                )
+                return ext_res
+
+            # Use provided template or default prompt
+            if template is not None:
+                prompt = self._serialize_template(template)
+            else:
+                prompt = "Extract all text and structured information from this document. Return as JSON."
+
+            # Process all images with VLM model
+            start_page, end_page = ext_res.input.limits.page_range
+            for i, image in enumerate(images):
+                # Calculate the actual page number based on the filtered range
+                page_number = start_page + i
+                try:
+                    predictions = list(self.vlm_model.process_images([image], prompt))
+
+                    if predictions:
+                        # Parse the extracted text as JSON if possible, otherwise use as-is
+                        extracted_text = predictions[0].text
+                        extracted_data = None
+
+                        try:
+                            extracted_data = json.loads(extracted_text)
+                        except (json.JSONDecodeError, ValueError):
+                            # If not valid JSON, keep extracted_data as None
+                            pass
+
+                        # Create page data with proper structure
+                        page_data = ExtractedPageData(
+                            page_no=page_number,
+                            extracted_data=extracted_data,
+                            raw_text=extracted_text,  # Always populate raw_text
+                        )
+                        ext_res.pages.append(page_data)
+                    else:
+                        # Add error page data
+                        page_data = ExtractedPageData(
+                            page_no=page_number,
+                            extracted_data=None,
+                            errors=["No extraction result from VLM model"],
+                        )
+                        ext_res.pages.append(page_data)
+
+                except Exception as e:
+                    _log.error(f"Error processing page {page_number}: {e}")
+                    page_data = ExtractedPageData(
+                        page_no=page_number, extracted_data=None, errors=[str(e)]
+                    )
+                    ext_res.pages.append(page_data)
+
+        except Exception as e:
+            _log.error(f"Error during extraction: {e}")
+            ext_res.errors.append(
+                ErrorItem(
+                    component_type="extraction_pipeline",
+                    module_name=self.__class__.__name__,
+                    error_message=str(e),
+                )
+            )
+
+        return ext_res
+
+    def _determine_status(self, ext_res: ExtractionResult) -> ConversionStatus:
+        """Determine the status based on extraction results."""
+        if ext_res.pages and not any(page.errors for page in ext_res.pages):
+            return ConversionStatus.SUCCESS
+        else:
+            return ConversionStatus.FAILURE
+
+    def _get_images_from_input(self, input_doc: InputDocument) -> list[Image]:
+        """Extract images from input document using the backend."""
+        images = []
+
+        try:
+            backend = input_doc._backend
+
+            assert isinstance(backend, PdfDocumentBackend)
+            # Use the backend's pagination interface
+            page_count = backend.page_count()
+
+            # Respect page range limits, following the same pattern as PaginatedPipeline
+            start_page, end_page = input_doc.limits.page_range
+            _log.info(
+                f"Processing pages {start_page}-{end_page} of {page_count} total pages for extraction"
+            )
+
+            for page_num in range(page_count):
+                # Only process pages within the specified range (0-based indexing)
+                if start_page - 1 <= page_num <= end_page - 1:
+                    try:
+                        page_backend = backend.load_page(page_num)
+                        if page_backend.is_valid():
+                            # Get page image at a reasonable scale
+                            page_image = page_backend.get_page_image(
+                                scale=self.pipeline_options.vlm_options.scale
+                            )
+                            images.append(page_image)
+                        else:
+                            _log.warning(f"Page {page_num + 1} backend is not valid")
+                    except Exception as e:
+                        _log.error(f"Error loading page {page_num + 1}: {e}")
+
+        except Exception as e:
+            _log.error(f"Error getting images from input document: {e}")
+
+        return images
+
+    def _serialize_template(self, template: ExtractionTemplateType) -> str:
+        """Serialize template to string based on its type."""
+        if isinstance(template, str):
+            return template
+        elif isinstance(template, dict):
+            return json.dumps(template, indent=2)
+        elif isinstance(template, BaseModel):
+            return template.model_dump_json(indent=2)
+        elif inspect.isclass(template) and issubclass(template, BaseModel):
+            from polyfactory.factories.pydantic_factory import ModelFactory
+
+            class ExtractionTemplateFactory(ModelFactory[template]):  # type: ignore
+                __use_examples__ = True  # prefer Field(examples=...) when present
+                __use_defaults__ = True  # use field defaults instead of random values
+
+            return ExtractionTemplateFactory.build().model_dump_json(indent=2)  # type: ignore
+        else:
+            raise ValueError(f"Unsupported template type: {type(template)}")
+
+    @classmethod
+    def get_default_options(cls) -> BaseOptions:
+        return VlmExtractionPipelineOptions()

docling/utils/model_downloader.py

@@ -4,6 +4,7 @@ from typing import Optional
 
 from docling.datamodel.layout_model_specs import DOCLING_LAYOUT_V2
 from docling.datamodel.pipeline_options import (
+    LayoutOptions,
     granite_picture_description,
     smolvlm_picture_description,
 )
@@ -47,7 +48,7 @@ def download_models(
     if with_layout:
         _log.info("Downloading layout model...")
         LayoutModel.download_models(
-            local_dir=output_dir / DOCLING_LAYOUT_V2.model_repo_folder,
+            local_dir=output_dir / LayoutOptions().model_spec.model_repo_folder,
             force=force,
             progress=progress,
         )

{docling-2.48.0.dist-info → docling-2.50.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docling
-Version: 2.48.0
+Version: 2.50.0
 Summary: SDK and CLI for parsing PDF, DOCX, HTML, and more, to a unified document representation for powering downstream workflows such as gen AI applications.
 Author-email: Christoph Auer <cau@zurich.ibm.com>, Michele Dolfi <dol@zurich.ibm.com>, Maxim Lysak <mly@zurich.ibm.com>, Nikos Livathinos <nli@zurich.ibm.com>, Ahmed Nassar <ahn@zurich.ibm.com>, Panos Vagenas <pva@zurich.ibm.com>, Peter Staar <taa@zurich.ibm.com>
 License-Expression: MIT
@@ -28,7 +28,7 @@ License-File: LICENSE
 Requires-Dist: pydantic<3.0.0,>=2.0.0
 Requires-Dist: docling-core[chunking]<3.0.0,>=2.42.0
 Requires-Dist: docling-parse<5.0.0,>=4.2.2
-Requires-Dist: docling-ibm-models<4,>=3.9.0
+Requires-Dist: docling-ibm-models<4,>=3.9.1
 Requires-Dist: filetype<2.0.0,>=1.2.0
 Requires-Dist: pypdfium2!=4.30.1,<5.0.0,>=4.30.0
 Requires-Dist: pydantic-settings<3.0.0,>=2.3.0
@@ -51,6 +51,7 @@ Requires-Dist: pluggy<2.0.0,>=1.0.0
 Requires-Dist: pylatexenc<3.0,>=2.10
 Requires-Dist: scipy<2.0.0,>=1.6.0
 Requires-Dist: accelerate<2,>=1.0.0
+Requires-Dist: polyfactory>=2.22.2
 Provides-Extra: tesserocr
 Requires-Dist: tesserocr<3.0.0,>=2.7.1; extra == "tesserocr"
 Provides-Extra: ocrmac
@@ -60,6 +61,7 @@ Requires-Dist: transformers<5.0.0,>=4.46.0; extra == "vlm"
 Requires-Dist: accelerate<2.0.0,>=1.2.1; extra == "vlm"
 Requires-Dist: mlx-vlm<1.0.0,>=0.3.0; (python_version >= "3.10" and sys_platform == "darwin" and platform_machine == "arm64") and extra == "vlm"
 Requires-Dist: vllm<1.0.0,>=0.10.0; (python_version >= "3.10" and sys_platform == "linux" and platform_machine == "x86_64") and extra == "vlm"
+Requires-Dist: qwen-vl-utils>=0.0.11; extra == "vlm"
 Provides-Extra: rapidocr
 Requires-Dist: rapidocr<4.0.0,>=3.3; python_version < "3.14" and extra == "rapidocr"
 Requires-Dist: onnxruntime<2.0.0,>=1.7.0; extra == "rapidocr"