hanlp-haystack 2.1.1__tar.gz → 2.2.0__tar.gz

{hanlp_haystack-2.1.1 → hanlp_haystack-2.2.0}/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [integrations/hanlp-v2.1.1] - 2026-01-27
+
+### 🧹 Chores
+
+- Pin transformers<5 for HanLP integration (#2785)
+
+
 ## [integrations/hanlp-v2.1.0] - 2026-01-14
 
 ### 🚀 Features

{hanlp_haystack-2.1.1 → hanlp_haystack-2.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hanlp-haystack
-Version: 2.1.1
+Version: 2.2.0
 Summary: An integration of Han Language Processing - HanLP as a ChineseDocumentSplitter component.
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/hanlp#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -15,12 +15,13 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.10
 Requires-Dist: hanlp>=2.1.1
 Requires-Dist: haystack-ai>=2.22.0
-Requires-Dist: tokenizers>=0.20.2
+Requires-Dist: tokenizers>=0.21.0
 Requires-Dist: transformers<5,>=4.1.1
 Description-Content-Type: text/markdown
 

hanlp_haystack-2.2.0/pydoc/config_docusaurus.yml

@@ -0,0 +1,13 @@
+loaders:
+  - modules:
+      - haystack_integrations.components.preprocessors.hanlp.chinese_document_splitter
+    search_path: [../src]
+processors:
+  - type: filter
+    documented_only: true
+    skip_empty_modules: true
+renderer:
+  description: HanLP integration for Haystack
+  id: integrations-hanlp
+  filename: hanlp.md
+  title: HanLP

{hanlp_haystack-2.1.1 → hanlp_haystack-2.2.0}/pyproject.toml

@@ -19,6 +19,7 @@ classifiers = [
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
   "Programming Language :: Python :: 3.13",
+  "Programming Language :: Python :: 3.14",
   "Programming Language :: Python :: Implementation :: CPython",
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
@@ -27,8 +28,8 @@ dependencies = [
   "hanlp>=2.1.1",
   # the following pins can be removed once HanLP supports transformers 5
   "transformers>=4.1.1,<5",
-  # tokenizers is a dependency of transformers. The following version is the first one which supports Python 3.13.
-  "tokenizers>=0.20.2",
+  # tokenizers is a dependency of transformers. The following version is the first one which supports Python 3.14.
+  "tokenizers>=0.21.0",
 ]
 
 [project.urls]
@@ -52,7 +53,7 @@ installer = "uv"
 dependencies = ["haystack-pydoc-tools", "ruff"]
 
 [tool.hatch.envs.default.scripts]
-docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
+docs = ["haystack-pydoc pydoc/config_docusaurus.yml"]
 fmt = "ruff check --fix {args}; ruff format {args}"
 fmt-check = "ruff check {args} && ruff format --check {args}"
 
@@ -70,7 +71,8 @@ dependencies = [
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
 all = 'pytest {args:tests}'
-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
+unit-cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x -m "not integration" {args:tests}'
+integration-cov-append-retry = 'pytest --cov=haystack_integrations --cov-append --reruns 3 --reruns-delay 30 -x -m "integration" {args:tests}'
 types = """mypy -p haystack_integrations.components.preprocessors.hanlp {args}"""
 
 [tool.mypy]
@@ -96,9 +98,17 @@ line-length = 120
 [tool.ruff.lint]
 select = [
   "A",
+  "ANN",
   "ARG",
   "B",
   "C",
+  "D102",   # Missing docstring in public method
+  "D103",   # Missing docstring in public function
+  "D205",   # 1 blank line required between summary line and description
+  "D209",   # Closing triple quotes go to new line
+  "D213",   # summary lines must be positioned on the second physical line of the docstring
+  "D417",   # Missing argument descriptions in the docstring
+  "D419",   # Docstring is empty
   "DTZ",
   "E",
   "EM",
@@ -139,6 +149,8 @@ ignore = [
   "ARG005",
   "RUF001",
   "RUF002",
+  # Allow `Any` - used legitimately for dynamic types and SDK boundaries
+  "ANN401",
 ]
 
 [tool.ruff.lint.isort]
@@ -149,11 +161,12 @@ ban-relative-imports = "parents"
 
 [tool.ruff.lint.per-file-ignores]
 # Tests can use magic values, assertions, and relative imports
-"tests/**/*" = ["PLR2004", "S101", "TID252"]
+"tests/**/*" = ["D", "PLR2004", "S101", "TID252", "ANN"]
 
 [tool.coverage.run]
 source = ["haystack_integrations"]
 branch = true
+relative_files = true
 parallel = false
 
 [tool.coverage.report]

{hanlp_haystack-2.1.1 → hanlp_haystack-2.2.0}/src/haystack_integrations/components/preprocessors/hanlp/chinese_document_splitter.py

@@ -4,6 +4,7 @@
 
 from collections.abc import Callable
 from copy import deepcopy
+from dataclasses import replace
 from typing import Any, Literal
 
 from haystack import Document, component, logging
@@ -440,10 +441,7 @@ class ChineseDocumentSplitter:
             previous_doc_start_idx = splits_start_idxs[i - 1]
             self._add_split_overlap_information(doc, doc_start_idx, previous_doc, previous_doc_start_idx)
 
-        for d in documents:
-            if d.content is not None:
-                d.content = d.content.replace(" ", "")
-        return documents
+        return [replace(d, content=d.content.replace(" ", "")) if d.content is not None else d for d in documents]
 
     @staticmethod
     def _add_split_overlap_information(

{hanlp_haystack-2.1.1 → hanlp_haystack-2.2.0}/tests/test_chinese_document_splitter.py

@@ -2,6 +2,8 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from unittest.mock import MagicMock, patch
+
 import pytest
 from haystack import Document
 from haystack.utils import deserialize_callable, serialize_callable
@@ -90,6 +92,145 @@ class TestChineseDocumentSplitter:
         assert callable(splitter.splitting_function)
         assert splitter.splitting_function("a.b.c") == ["a", "b", "c"]
 
+    def test_validate_init_parameters(self):
+        ChineseDocumentSplitter._validate_init_parameters(
+            split_by="word",
+            split_length=1000,
+            split_overlap=200,
+            split_threshold=0,
+            granularity="coarse",
+        )
+
+        with pytest.raises(ValueError, match="split_length must be positive"):
+            ChineseDocumentSplitter._validate_init_parameters(split_length=0)
+
+        with pytest.raises(ValueError, match="split_overlap must be non-negative"):
+            ChineseDocumentSplitter._validate_init_parameters(split_overlap=-1)
+
+        with pytest.raises(ValueError, match="split_overlap must be less than split_length"):
+            ChineseDocumentSplitter._validate_init_parameters(split_overlap=1000, split_length=500)
+
+        with pytest.raises(ValueError, match="split_threshold must be non-negative"):
+            ChineseDocumentSplitter._validate_init_parameters(split_threshold=-1)
+
+        with pytest.raises(ValueError, match="split_threshold must be less than split_length"):
+            ChineseDocumentSplitter._validate_init_parameters(split_threshold=1001, split_length=1000)
+
+        with pytest.raises(
+            ValueError,
+            match="split_by must be one of 'word', 'sentence', 'passage', 'page', 'line', 'period', 'function'",
+        ):
+            ChineseDocumentSplitter._validate_init_parameters(split_by="invalid")
+
+        with pytest.raises(ValueError, match="granularity must be one of 'coarse', 'fine'"):
+            ChineseDocumentSplitter._validate_init_parameters(granularity="invalid")
+
+    def test_split_by_character_returns_empty_for_none_content(self):
+        splitter = ChineseDocumentSplitter(split_by="word")
+        assert splitter._split_by_character(Document(content=None)) == []
+
+    def test_split_by_hanlp_sentence_returns_empty_for_none_content(self):
+        splitter = ChineseDocumentSplitter(split_by="sentence")
+        assert splitter._split_by_hanlp_sentence(Document(content=None)) == []
+
+    def test_split_document_dispatches_to_function_splitter(self):
+        splitter = ChineseDocumentSplitter(split_by="function", splitting_function=custom_split)
+        doc = Document(content="a.b.c")
+        result = splitter._split_document(doc)
+        assert [d.content for d in result] == ["a", "b", "c"]
+
+    def test_split_by_function_returns_empty_for_none_content(self):
+        splitter = ChineseDocumentSplitter(split_by="function", splitting_function=custom_split)
+        assert splitter._split_by_function(Document(content=None)) == []
+
+    def test_split_by_function_raises_when_no_function_provided(self):
+        splitter = ChineseDocumentSplitter(split_by="function", splitting_function=custom_split)
+        splitter.splitting_function = None
+        with pytest.raises(ValueError, match=r"No splitting function provided\."):
+            splitter._split_by_function(Document(content="a.b"))
+
+    def test_split_by_function_raises_when_function_returns_non_list(self):
+        splitter = ChineseDocumentSplitter(split_by="function", splitting_function=lambda t: "not a list")
+        with pytest.raises(ValueError, match="must return a list of strings"):
+            splitter._split_by_function(Document(content="a.b"))
+
+    def test_split_by_function_produces_documents_with_metadata(self):
+        splitter = ChineseDocumentSplitter(split_by="function", splitting_function=custom_split)
+        doc = Document(content="a.b.c", meta={"name": "src"})
+        result = splitter._split_by_function(doc)
+        assert len(result) == 3
+        assert [d.content for d in result] == ["a", "b", "c"]
+        assert all(d.meta["source_id"] == doc.id for d in result)
+        assert all(d.meta["name"] == "src" for d in result)
+        assert [d.meta["split_id"] for d in result] == [0, 1, 2]
+
+    @pytest.mark.parametrize(
+        ("previous_content", "current_content"),
+        [(None, "abc"), ("abc", None)],
+    )
+    def test_add_split_overlap_information_skips_when_content_is_none(self, previous_content, current_content):
+        previous_doc = Document(content=previous_content, meta={"_split_overlap": []})
+        current_doc = Document(content=current_content, meta={"_split_overlap": []})
+        ChineseDocumentSplitter._add_split_overlap_information(current_doc, 0, previous_doc, 0)
+        assert current_doc.meta["_split_overlap"] == []
+        assert previous_doc.meta["_split_overlap"] == []
+
+    def test_add_split_overlap_information_skips_when_no_matching_prefix(self):
+        previous_doc = Document(content="abcdef", meta={"_split_overlap": []})
+        current_doc = Document(content="xyz", meta={"_split_overlap": []})
+        ChineseDocumentSplitter._add_split_overlap_information(current_doc, 2, previous_doc, 0)
+        assert current_doc.meta["_split_overlap"] == []
+        assert previous_doc.meta["_split_overlap"] == []
+
+    @pytest.mark.parametrize("elements", [[], ["", "   ", "\t"]])
+    def test_concatenate_units_returns_empty_for_empty_or_whitespace(self, elements):
+        splitter = ChineseDocumentSplitter(split_by="word", split_length=5, split_overlap=1)
+        assert splitter._concatenate_units(elements, split_length=5, split_overlap=1, split_threshold=0) == ([], [], [])
+
+    def test_concatenate_units_returns_single_chunk_for_short_input(self):
+        splitter = ChineseDocumentSplitter(split_by="word", split_length=10, split_overlap=0)
+        text_splits, pages, start_idxs = splitter._concatenate_units(
+            ["a", "b", "c"], split_length=10, split_overlap=0, split_threshold=0
+        )
+        assert text_splits == ["abc"]
+        assert pages == [1]
+        assert start_idxs == [0]
+
+    def test_concatenate_units_applies_split_threshold(self):
+        splitter = ChineseDocumentSplitter(split_by="word", split_length=2, split_overlap=0)
+        text_splits, _, _ = splitter._concatenate_units(
+            ["a", "b", "c"], split_length=2, split_overlap=0, split_threshold=2
+        )
+        # the trailing "c" chunk is smaller than split_threshold=2 and gets attached to previous
+        assert text_splits == ["abc"]
+
+    def test_number_of_sentences_to_keep_returns_zero_when_overlap_is_zero(self):
+        splitter = ChineseDocumentSplitter(split_by="word", split_length=10, split_overlap=0)
+        assert splitter._number_of_sentences_to_keep(["s1", "s2", "s3"], split_length=10, split_overlap=0) == 0
+
+    def test_number_of_sentences_to_keep_stops_when_overlap_reached(self):
+        splitter = ChineseDocumentSplitter(split_by="word", split_length=10, split_overlap=2)
+        # one word per sentence, so we need at least 3 sentences to exceed split_overlap=2
+        splitter.chinese_tokenizer = MagicMock(side_effect=lambda s: [s])
+        result = splitter._number_of_sentences_to_keep(["s1", "s2", "s3", "s4"], split_length=10, split_overlap=2)
+        assert result == 3
+
+    def test_number_of_sentences_to_keep_stops_when_split_length_exceeded(self):
+        splitter = ChineseDocumentSplitter(split_by="word", split_length=2, split_overlap=1)
+        # each sentence tokenizes to 5 "words", so first iteration already exceeds split_length
+        splitter.chinese_tokenizer = MagicMock(return_value=["a", "b", "c", "d", "e"])
+        assert splitter._number_of_sentences_to_keep(["s1", "s2", "s3"], split_length=2, split_overlap=1) == 0
+
+    def test_warm_up_is_idempotent(self):
+        splitter = ChineseDocumentSplitter(split_by="word")
+        with patch(
+            "haystack_integrations.components.preprocessors.hanlp.chinese_document_splitter.hanlp"
+        ) as mock_hanlp:
+            mock_hanlp.load.return_value = MagicMock()
+            splitter.warm_up()
+            splitter.warm_up()
+            assert mock_hanlp.load.call_count == 2  # once for tokenizer, once for split_sent
+
     @pytest.mark.integration
     def test_empty_list(self):
         splitter = ChineseDocumentSplitter()
@@ -216,36 +357,3 @@ class TestChineseDocumentSplitter:
                 f"Chunks {i} and {i + 1} do not overlap. "
                 f"Tail (up to 20 chars): '{overlap_prev}' vs Head (up to 20 chars): '{overlap_curr}'"
             )
-
-    def test_validate_init_parameters(self):
-        ChineseDocumentSplitter._validate_init_parameters(
-            split_by="word",
-            split_length=1000,
-            split_overlap=200,
-            split_threshold=0,
-            granularity="coarse",
-        )
-
-        with pytest.raises(ValueError, match="split_length must be positive"):
-            ChineseDocumentSplitter._validate_init_parameters(split_length=0)
-
-        with pytest.raises(ValueError, match="split_overlap must be non-negative"):
-            ChineseDocumentSplitter._validate_init_parameters(split_overlap=-1)
-
-        with pytest.raises(ValueError, match="split_overlap must be less than split_length"):
-            ChineseDocumentSplitter._validate_init_parameters(split_overlap=1000, split_length=500)
-
-        with pytest.raises(ValueError, match="split_threshold must be non-negative"):
-            ChineseDocumentSplitter._validate_init_parameters(split_threshold=-1)
-
-        with pytest.raises(ValueError, match="split_threshold must be less than split_length"):
-            ChineseDocumentSplitter._validate_init_parameters(split_threshold=1001, split_length=1000)
-
-        with pytest.raises(
-            ValueError,
-            match="split_by must be one of 'word', 'sentence', 'passage', 'page', 'line', 'period', 'function'",
-        ):
-            ChineseDocumentSplitter._validate_init_parameters(split_by="invalid")
-
-        with pytest.raises(ValueError, match="granularity must be one of 'coarse', 'fine'"):
-            ChineseDocumentSplitter._validate_init_parameters(granularity="invalid")

hanlp_haystack-2.1.1/pydoc/config_docusaurus.yml

@@ -1,28 +0,0 @@
-loaders:
-- ignore_when_discovered:
-  - __init__
-  modules:
-  - haystack_integrations.components.preprocessors.hanlp.chinese_document_splitter
-  search_path:
-  - ../src
-  type: haystack_pydoc_tools.loaders.CustomPythonLoader
-processors:
-- do_not_filter_modules: false
-  documented_only: true
-  expression: null
-  skip_empty_modules: true
-  type: filter
-- type: smart
-- type: crossref
-renderer:
-  description: HanLP integration for Haystack
-  id: integrations-hanlp
-  markdown:
-    add_member_class_prefix: false
-    add_method_class_prefix: true
-    classdef_code_block: false
-    descriptive_class_title: false
-    descriptive_module_title: true
-    filename: hanlp.md
-  title: HanLP
-  type: haystack_pydoc_tools.renderers.DocusaurusRenderer