docling-haystack 0.3.0__tar.gz → 1.0.0__tar.gz

docling_haystack-1.0.0/CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+
+## [integrations/docling-v0.4.0] - 2026-05-04
+
+### 🚀 Features
+
+- Add serialization/deserialization to DoclingConverter  (#3267)
+
+### 🧪 Testing
+
+- Docling - add a few unit tests (#3212)
+
+
+## [integrations/docling-v0.3.0] - 2026-04-10
+
+### 🚀 Features
+
+- (docling) Drop temp files for ByteStream sources (#3130)
+
+
+## [integrations/docling-v0.2.0] - 2026-04-08
+
+### 🚀 Features
+
+- Add Docling document converter (#3066)
+
+### 🚜 Refactor
+
+- *(docling)* Add meta parameter to run(); introduce sources; deprecate paths (#3103)
+
+<!-- generated by git-cliff -->

{docling_haystack-0.3.0 → docling_haystack-1.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: docling-haystack
-Version: 0.3.0
+Version: 1.0.0
 Summary: Haystack integration for docling
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/docling#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues

{docling_haystack-0.3.0 → docling_haystack-1.0.0}/src/haystack_integrations/components/converters/docling/converter.py

@@ -10,14 +10,18 @@ from pathlib import Path
 from typing import Any
 
 from docling_core.types.io import DocumentStream
-from haystack import Document, component
+from haystack import Document, component, logging
 from haystack.components.converters.utils import normalize_metadata
+from haystack.core.serialization import default_from_dict, default_to_dict
 from haystack.dataclasses import ByteStream
+from haystack.utils.base_serialization import deserialize_class_instance, serialize_class_instance
 
 from docling.chunking import BaseChunk, BaseChunker, HybridChunker
 from docling.datamodel.document import DoclingDocument
 from docling.document_converter import DocumentConverter
 
+logger = logging.getLogger(__name__)
+
 
 def _bytestream_to_document_stream(source: ByteStream) -> DocumentStream:
     """
@@ -63,13 +67,27 @@ class BaseMetaExtractor(ABC):
         """Extract Docling document meta."""
         raise NotImplementedError()
 
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize to a dictionary."""
+        return {}
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "BaseMetaExtractor":  # noqa: ARG003
+        """Deserialize from a dictionary."""
+        return cls()
+
 
 class MetaExtractor(BaseMetaExtractor):
     """MetaExtractor."""
 
     def extract_chunk_meta(self, chunk: BaseChunk) -> dict[str, Any]:
         """Extract chunk meta."""
-        return {"dl_meta": chunk.export_json_dict()}
+        meta: dict[str, Any] = {"dl_meta": chunk.export_json_dict()}
+        doc_items = getattr(chunk.meta, "doc_items", [])
+        page_nos = {prov.page_no for item in doc_items for prov in getattr(item, "prov", [])}
+        if page_nos:
+            meta["page_number"] = min(page_nos)
+        return meta
 
     def extract_dl_doc_meta(self, dl_doc: DoclingDocument) -> dict[str, Any]:
         """Extract Docling document meta."""
@@ -84,7 +102,7 @@ class DoclingConverter:
         self,
         converter: DocumentConverter | None = None,
         convert_kwargs: dict[str, Any] | None = None,
-        export_type: ExportType = ExportType.DOC_CHUNKS,
+        export_type: ExportType = ExportType.MARKDOWN,
         md_export_kwargs: dict[str, Any] | None = None,
         chunker: BaseChunker | None = None,
         meta_extractor: BaseMetaExtractor | None = None,
@@ -97,10 +115,10 @@ class DoclingConverter:
         :param convert_kwargs: Any parameters to pass to Docling conversion; if not set, a
             system default is used.
         :param export_type: The export mode to use:
-            * `ExportType.MARKDOWN` captures each input document as a single
+            * `ExportType.MARKDOWN` (default) captures each input document as a single
               markdown `Document`.
-            * `ExportType.DOC_CHUNKS` (default) first chunks each input document
-              and then returns one `Document` per chunk.
+            * `ExportType.DOC_CHUNKS` first chunks each input document and then returns
+              one `Document` per chunk.
             * `ExportType.JSON` serializes the full Docling document to a JSON string.
         :param md_export_kwargs: Any parameters to pass to Markdown export (applicable in
             case of `ExportType.MARKDOWN`).
@@ -123,6 +141,53 @@ class DoclingConverter:
             self._chunker_instance = chunker or HybridChunker()
         self._meta_extractor_instance = meta_extractor or MetaExtractor()
 
+    def to_dict(self) -> dict[str, Any]:
+        """Serialize this component to a dictionary."""
+        if self.converter is not None:
+            logger.warning(
+                "DoclingConverter.to_dict: the 'converter' parameter cannot be serialized and will be dropped. "
+                "The component will use the default DocumentConverter when restored from the serialized form."
+            )
+        if self.chunker is not None:
+            logger.warning(
+                "DoclingConverter.to_dict: the 'chunker' parameter cannot be serialized and will be dropped. "
+                "The component will use the default chunker when restored from the serialized form."
+            )
+
+        meta_extractor_data = None
+        if self.meta_extractor is not None:
+            meta_extractor_data = serialize_class_instance(self.meta_extractor)
+
+        return default_to_dict(
+            self,
+            converter=None,
+            convert_kwargs=self.convert_kwargs,
+            export_type=self.export_type.value,
+            md_export_kwargs=self.md_export_kwargs,
+            chunker=None,
+            meta_extractor=meta_extractor_data,
+        )
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "DoclingConverter":
+        """
+        Deserialize this component from a dictionary.
+
+        The `converter` and `chunker` parameters are not serializable and are always ignored during
+        deserialization; the restored instance will use the default `DocumentConverter` and `HybridChunker`
+        respectively.
+
+        :param data: Dictionary with keys `type` and `init_parameters`, as produced by `to_dict`.
+        :returns: A new `DoclingConverter` instance.
+        """
+        init_params = data.get("init_parameters", {})
+
+        meta_extractor_data = init_params.get("meta_extractor")
+        if meta_extractor_data is not None:
+            init_params["meta_extractor"] = deserialize_class_instance(meta_extractor_data)
+
+        return default_from_dict(cls, data)
+
     @component.output_types(documents=list[Document])
     def run(
         self,
@@ -174,15 +239,17 @@ class DoclingConverter:
                 merged_meta = source_meta
 
             if self.export_type == ExportType.DOC_CHUNKS:
-                chunk_iter = self._chunker_instance.chunk(dl_doc=dl_doc)
-                hs_docs = [
-                    Document(
-                        content=self._chunker_instance.contextualize(chunk=chunk),
-                        meta={**self._meta_extractor_instance.extract_chunk_meta(chunk=chunk), **merged_meta},
-                    )
-                    for chunk in chunk_iter
-                ]
-                documents.extend(hs_docs)
+                split_idx_start = 0
+                for split_id, chunk in enumerate(self._chunker_instance.chunk(dl_doc=dl_doc)):
+                    content = self._chunker_instance.contextualize(chunk=chunk)
+                    meta = {
+                        **self._meta_extractor_instance.extract_chunk_meta(chunk=chunk),
+                        "split_id": split_id,
+                        "split_idx_start": split_idx_start,
+                        **merged_meta,
+                    }
+                    documents.append(Document(content=content, meta=meta))
+                    split_idx_start += len(chunk.text)
             elif self.export_type == ExportType.MARKDOWN:
                 hs_doc = Document(
                     content=dl_doc.export_to_markdown(**self.md_export_kwargs),

{docling_haystack-0.3.0 → docling_haystack-1.0.0}/tests/test_converter.py

@@ -7,11 +7,16 @@ from typing import Any
 from unittest.mock import MagicMock
 
 import pytest
+from docling.chunking import HybridChunker
+from docling.document_converter import DocumentConverter
 from docling_core.types.io import DocumentStream
-from haystack.core.serialization import component_from_dict, component_to_dict
 from haystack.dataclasses import ByteStream
 
-from haystack_integrations.components.converters.docling import DoclingConverter, ExportType
+from haystack_integrations.components.converters.docling import (
+    DoclingConverter,
+    ExportType,
+    MetaExtractor,
+)
 from haystack_integrations.components.converters.docling.converter import _bytestream_to_document_stream
 
 
@@ -54,7 +59,7 @@ def test_run_doc_chunks_minimal() -> None:
 
     assert "contextualized-chunk-1-of-dl-doc-for-file-a.pdf" in contents
     assert "contextualized-chunk-2-of-dl-doc-for-file-a.pdf" in contents
-    assert {"chunk_id": "chunk-1-of-dl-doc-for-file-a.pdf"} in metas
+    assert any(m.get("chunk_id") == "chunk-1-of-dl-doc-for-file-a.pdf" for m in metas)
 
     # Ensure our collaborators were actually exercised.
     assert converter_mock.convert.call_count == len(paths)
@@ -130,8 +135,6 @@ def test_run_json_minimal() -> None:
 
 
 def test_legacy_import_path() -> None:
-    import warnings
-
     with warnings.catch_warnings(record=True) as caught:
         warnings.simplefilter("always")
         from docling_haystack.converter import DoclingConverter as LegacyDoclingConverter
@@ -142,63 +145,59 @@ def test_legacy_import_path() -> None:
     )
 
 
-def test_component_from_dict_legacy_nulls() -> None:
-    # Before the public-attribute refactor, default serialization couldn't find
-    # the _-prefixed attributes and fell back to the init defaults, so
-    # convert_kwargs and md_export_kwargs were always serialized as null.
-    # Verify that such a serialized dict still deserializes correctly.
-    legacy_data = {
+def test_component_to_dict_defaults() -> None:
+    converter = DoclingConverter()
+    assert converter.to_dict() == {
         "type": "haystack_integrations.components.converters.docling.converter.DoclingConverter",
         "init_parameters": {
             "converter": None,
-            "convert_kwargs": None,
-            "export_type": "doc_chunks",
-            "md_export_kwargs": None,
+            "convert_kwargs": {},
+            "export_type": "markdown",
+            "md_export_kwargs": {"image_placeholder": ""},
             "chunker": None,
             "meta_extractor": None,
         },
     }
-    restored = component_from_dict(DoclingConverter, legacy_data, "docling_converter")
-
-    assert restored.convert_kwargs == {}
-    assert restored.md_export_kwargs == {"image_placeholder": ""}
-    assert restored.export_type == ExportType.DOC_CHUNKS
-    assert restored.converter is None
-    assert restored.chunker is None
-    assert restored.meta_extractor is None
-
-
-def test_component_to_dict_defaults() -> None:
-    converter = DoclingConverter()
-    data = component_to_dict(converter, "docling_converter")
-
-    init_params = data["init_parameters"]
-    assert init_params["converter"] is None
-    assert init_params["convert_kwargs"] == {}
-    assert init_params["export_type"] == ExportType.DOC_CHUNKS
-    assert init_params["md_export_kwargs"] == {"image_placeholder": ""}
-    assert init_params["chunker"] is None
-    assert init_params["meta_extractor"] is None
 
 
 def test_component_to_dict_custom_params() -> None:
     converter = DoclingConverter(
+        converter=DocumentConverter(),
         convert_kwargs={"raises_on_error": False},
         export_type=ExportType.MARKDOWN,
         md_export_kwargs={"image_placeholder": "[img]"},
+        meta_extractor=MetaExtractor(),
     )
-    data = component_to_dict(converter, "docling_converter")
-
-    init_params = data["init_parameters"]
-    assert init_params["convert_kwargs"] == {"raises_on_error": False}
-    assert init_params["export_type"] == ExportType.MARKDOWN
-    assert init_params["md_export_kwargs"] == {"image_placeholder": "[img]"}
+    assert converter.to_dict() == {
+        "type": "haystack_integrations.components.converters.docling.converter.DoclingConverter",
+        "init_parameters": {
+            "converter": None,
+            "convert_kwargs": {"raises_on_error": False},
+            "export_type": "markdown",
+            "md_export_kwargs": {"image_placeholder": "[img]"},
+            "chunker": None,
+            "meta_extractor": {
+                "type": "haystack_integrations.components.converters.docling.converter.MetaExtractor",
+                "data": {},
+            },
+        },
+    }
 
 
 def test_component_from_dict_defaults() -> None:
-    converter = DoclingConverter()
-    data = component_to_dict(converter, "docling_converter")
-    restored = component_from_dict(DoclingConverter, data, "docling_converter")
+    # null kwargs mirror the pre-refactor serialization format and must still deserialize correctly
+    data = {
+        "type": "haystack_integrations.components.converters.docling.converter.DoclingConverter",
+        "init_parameters": {
+            "converter": None,
+            "convert_kwargs": None,
+            "export_type": "doc_chunks",
+            "md_export_kwargs": None,
+            "chunker": None,
+            "meta_extractor": None,
+        },
+    }
+    restored = DoclingConverter.from_dict(data)
 
     assert restored.converter is None
     assert restored.convert_kwargs == {}
@@ -209,17 +208,44 @@ def test_component_from_dict_defaults() -> None:
 
 
 def test_component_from_dict_custom_params() -> None:
-    converter = DoclingConverter(
-        convert_kwargs={"raises_on_error": False},
-        export_type=ExportType.JSON,
-        md_export_kwargs={"image_placeholder": "[img]"},
-    )
-    data = component_to_dict(converter, "docling_converter")
-    restored = component_from_dict(DoclingConverter, data, "docling_converter")
+    data = {
+        "type": "haystack_integrations.components.converters.docling.converter.DoclingConverter",
+        "init_parameters": {
+            "converter": None,
+            "convert_kwargs": {"raises_on_error": False},
+            "export_type": "json",
+            "md_export_kwargs": {"image_placeholder": "[img]"},
+            "chunker": None,
+            "meta_extractor": {
+                "type": "haystack_integrations.components.converters.docling.converter.MetaExtractor",
+                "data": {},
+            },
+        },
+    }
+    restored = DoclingConverter.from_dict(data)
 
+    assert restored.converter is None
     assert restored.convert_kwargs == {"raises_on_error": False}
     assert restored.export_type == ExportType.JSON
     assert restored.md_export_kwargs == {"image_placeholder": "[img]"}
+    assert restored.chunker is None
+    assert isinstance(restored.meta_extractor, MetaExtractor)
+
+
+def test_component_to_dict_chunker_warns_and_is_dropped() -> None:
+    converter = DoclingConverter(export_type=ExportType.DOC_CHUNKS, chunker=HybridChunker(merge_peers=False))
+
+    assert converter.to_dict() == {
+        "type": "haystack_integrations.components.converters.docling.converter.DoclingConverter",
+        "init_parameters": {
+            "converter": None,
+            "convert_kwargs": {},
+            "export_type": "doc_chunks",
+            "md_export_kwargs": {"image_placeholder": ""},
+            "chunker": None,
+            "meta_extractor": None,
+        },
+    }
 
 
 def test_run_with_sources_parameter() -> None:
@@ -441,3 +467,152 @@ class TestBytestreamToDocumentStream:
         ds = _bytestream_to_document_stream(bs)
         assert isinstance(ds, DocumentStream)
         assert isinstance(ds.stream, BytesIO)
+
+    def test_unknown_mime_type_keeps_base_name(self) -> None:
+        # mimetypes.guess_extension returns None for unknown types, so the name stays as-is.
+        assert mimetypes.guess_extension("application/x-totally-made-up-type") is None
+        bs = ByteStream(
+            data=b"data",
+            meta={"file_path": "report"},
+            mime_type="application/x-totally-made-up-type",
+        )
+        ds = _bytestream_to_document_stream(bs)
+        assert ds.name == "report"
+
+
+class TestMetaExtractor:
+    def test_extract_chunk_meta_wraps_export_json_dict(self) -> None:
+        chunk = MagicMock()
+        chunk.export_json_dict.return_value = {"some": "dict"}
+        chunk.meta.doc_items = []
+
+        result = MetaExtractor().extract_chunk_meta(chunk=chunk)
+
+        assert result == {"dl_meta": {"some": "dict"}}
+        chunk.export_json_dict.assert_called_once_with()
+
+    def test_extract_chunk_meta_includes_page_number(self) -> None:
+        prov = MagicMock()
+        prov.page_no = 3
+        doc_item = MagicMock()
+        doc_item.prov = [prov]
+
+        chunk = MagicMock()
+        chunk.export_json_dict.return_value = {"some": "dict"}
+        chunk.meta.doc_items = [doc_item]
+
+        result = MetaExtractor().extract_chunk_meta(chunk=chunk)
+
+        assert result == {"dl_meta": {"some": "dict"}, "page_number": 3}
+
+    def test_extract_chunk_meta_page_number_uses_minimum(self) -> None:
+        prov1 = MagicMock()
+        prov1.page_no = 5
+        prov2 = MagicMock()
+        prov2.page_no = 3
+        doc_item = MagicMock()
+        doc_item.prov = [prov1, prov2]
+
+        chunk = MagicMock()
+        chunk.export_json_dict.return_value = {}
+        chunk.meta.doc_items = [doc_item]
+
+        result = MetaExtractor().extract_chunk_meta(chunk=chunk)
+
+        assert result["page_number"] == 3
+
+    def test_extract_dl_doc_meta_with_origin(self) -> None:
+        dl_doc = MagicMock()
+        dl_doc.origin.model_dump.return_value = {"filename": "foo.pdf", "mimetype": "application/pdf"}
+
+        result = MetaExtractor().extract_dl_doc_meta(dl_doc=dl_doc)
+
+        assert result == {"dl_meta": {"origin": {"filename": "foo.pdf", "mimetype": "application/pdf"}}}
+        dl_doc.origin.model_dump.assert_called_once_with(exclude_none=True)
+
+    def test_extract_dl_doc_meta_without_origin(self) -> None:
+        dl_doc = MagicMock()
+        dl_doc.origin = None
+
+        result = MetaExtractor().extract_dl_doc_meta(dl_doc=dl_doc)
+
+        assert result == {}
+
+
+def test_run_without_sources_or_paths_raises_value_error() -> None:
+    converter = DoclingConverter(converter=MagicMock(), meta_extractor=MagicMock())
+    with pytest.raises(ValueError, match=r"Either 'sources' or the deprecated 'paths' parameter must be provided."):
+        converter.run()
+
+
+def test_run_doc_chunks_split_id_and_split_idx_start() -> None:
+    converter_mock = MagicMock()
+    chunker_mock = MagicMock()
+    meta_extractor_mock = MagicMock()
+
+    converter_mock.convert.return_value = SimpleNamespace(document="dl-doc")
+
+    chunks = [
+        SimpleNamespace(text="hello world"),
+        SimpleNamespace(text="foo bar baz"),
+    ]
+    chunker_mock.chunk.return_value = chunks
+    chunker_mock.contextualize.side_effect = lambda chunk: f"ctx:{chunk.text}"
+    meta_extractor_mock.extract_chunk_meta.return_value = {}
+
+    converter = DoclingConverter(
+        converter=converter_mock,
+        export_type=ExportType.DOC_CHUNKS,
+        chunker=chunker_mock,
+        meta_extractor=meta_extractor_mock,
+    )
+
+    result = converter.run(sources=["doc.pdf"])
+    documents = result["documents"]
+
+    assert len(documents) == 2
+    assert documents[0].meta["split_id"] == 0
+    assert documents[0].meta["split_idx_start"] == 0
+    assert documents[1].meta["split_id"] == 1
+    assert documents[1].meta["split_idx_start"] == len("hello world")
+
+
+def test_run_doc_chunks_split_id_resets_per_document() -> None:
+    converter_mock = MagicMock()
+    chunker_mock = MagicMock()
+    meta_extractor_mock = MagicMock()
+
+    converter_mock.convert.side_effect = [
+        SimpleNamespace(document="dl-doc-a"),
+        SimpleNamespace(document="dl-doc-b"),
+    ]
+    chunker_mock.chunk.side_effect = lambda dl_doc: [
+        SimpleNamespace(text=f"chunk-1-of-{dl_doc}"),
+        SimpleNamespace(text=f"chunk-2-of-{dl_doc}"),
+    ]
+    chunker_mock.contextualize.side_effect = lambda chunk: chunk.text
+    meta_extractor_mock.extract_chunk_meta.return_value = {}
+
+    converter = DoclingConverter(
+        converter=converter_mock,
+        export_type=ExportType.DOC_CHUNKS,
+        chunker=chunker_mock,
+        meta_extractor=meta_extractor_mock,
+    )
+
+    result = converter.run(sources=["a.pdf", "b.pdf"])
+    documents = result["documents"]
+
+    # split_id and split_idx_start reset for each source document
+    doc_a_chunks = documents[:2]
+    doc_b_chunks = documents[2:]
+
+    assert doc_a_chunks[0].meta["split_id"] == 0
+    assert doc_a_chunks[0].meta["split_idx_start"] == 0
+    assert doc_a_chunks[1].meta["split_id"] == 1
+    assert doc_a_chunks[1].meta["split_idx_start"] == len("chunk-1-of-dl-doc-a")
+
+    assert doc_b_chunks[0].meta["split_id"] == 0
+    assert doc_b_chunks[0].meta["split_idx_start"] == 0
+    assert doc_b_chunks[1].meta["split_id"] == 1
+    assert doc_b_chunks[1].meta["split_idx_start"] == len("chunk-1-of-dl-doc-b")

docling_haystack-0.3.0/CHANGELOG.md

@@ -1,13 +0,0 @@
-# Changelog
-
-## [integrations/docling-v0.2.0] - 2026-04-08
-
-### 🚀 Features
-
-- Add Docling document converter (#3066)
-
-### 🚜 Refactor
-
-- *(docling)* Add meta parameter to run(); introduce sources; deprecate paths (#3103)
-
-<!-- generated by git-cliff -->