hanlp-haystack 2.0.0__tar.gz → 2.1.1__tar.gz

hanlp_haystack-2.1.1/CHANGELOG.md

@@ -0,0 +1,51 @@
+# Changelog
+
+## [integrations/hanlp-v2.1.0] - 2026-01-14
+
+### 🚀 Features
+
+- Update hanlp components to auto call run `warm_up` (#2749)
+
+
+## [integrations/hanlp-v2.0.0] - 2026-01-12
+
+### 🐛 Bug Fixes
+
+- Docs: fix hanlp markdown filenames (#2480)
+
+### 📚 Documentation
+
+- Add pydoc configurations for Docusaurus (#2411)
+
+### ⚙️ CI
+
+- Change pytest command (#2475)
+
+### 🧹 Chores
+
+- Remove Readme API CI workflow and configs (#2573)
+- Make fmt command more forgiving (#2671)
+- [**breaking**] Hanlp - drop Python 3.9 and use X|Y typing (#2707)
+
+### 🌀 Miscellaneous
+
+- Enhancement: Adopt PEP 585 type hinting (part 4) (#2527)
+
+## [integrations/hanlp-v1.0.1] - 2025-10-14
+
+### 🐛 Bug Fixes
+
+- Missing decorator output for ChineseDocumentSplitter (#2377)
+
+
+### 🧹 Chores
+
+- Standardize readmes - part 2 (#2205)
+
+## [integrations/hanlp-v1.0.0] - 2025-07-22
+
+### 🚀 Features
+
+- Add new HanLP integration with ChineseDocumentSplitter (#1943)
+
+<!-- generated by git-cliff -->

{hanlp_haystack-2.0.0 → hanlp_haystack-2.1.1}/LICENSE.txt

@@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
 
 To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
 
-Copyright [yyyy] [name of copyright owner]
+Copyright 2025 deepset GmbH
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

{hanlp_haystack-2.0.0 → hanlp_haystack-2.1.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hanlp-haystack
-Version: 2.0.0
+Version: 2.1.1
 Summary: An integration of Han Language Processing - HanLP as a ChineseDocumentSplitter component.
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/hanlp#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -20,6 +20,8 @@ Classifier: Programming Language :: Python :: Implementation :: PyPy
 Requires-Python: >=3.10
 Requires-Dist: hanlp>=2.1.1
 Requires-Dist: haystack-ai>=2.22.0
+Requires-Dist: tokenizers>=0.20.2
+Requires-Dist: transformers<5,>=4.1.1
 Description-Content-Type: text/markdown
 
 # hanlp-haystack

{hanlp_haystack-2.0.0 → hanlp_haystack-2.1.1}/pyproject.toml

@@ -24,7 +24,11 @@ classifiers = [
 ]
 dependencies = [
   "haystack-ai>=2.22.0",
-  "hanlp>=2.1.1"
+  "hanlp>=2.1.1",
+  # the following pins can be removed once HanLP supports transformers 5
+  "transformers>=4.1.1,<5",
+  # tokenizers is a dependency of transformers. The following version is the first one which supports Python 3.13.
+  "tokenizers>=0.20.2",
 ]
 
 [project.urls]

{hanlp_haystack-2.0.0 → hanlp_haystack-2.1.1}/src/haystack_integrations/components/preprocessors/hanlp/__init__.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2025-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
+
 from .chinese_document_splitter import ChineseDocumentSplitter
 
 __all__ = [

{hanlp_haystack-2.0.0 → hanlp_haystack-2.1.1}/src/haystack_integrations/components/preprocessors/hanlp/chinese_document_splitter.py

@@ -12,6 +12,7 @@ from haystack.utils import deserialize_callable, serialize_callable
 from more_itertools import windowed
 
 import hanlp
+from hanlp.common.component import Component as HanlpComponent
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +48,6 @@ class ChineseDocumentSplitter:
     splitter = ChineseDocumentSplitter(
         split_by="word", split_length=10, split_overlap=3, respect_sentence_boundary=True
     )
-    splitter.warm_up()
     result = splitter.run(documents=[doc])
     print(result["documents"])
     ```
@@ -62,7 +62,7 @@ class ChineseDocumentSplitter:
         respect_sentence_boundary: bool = False,
         splitting_function: Callable | None = None,
         granularity: Literal["coarse", "fine"] = "coarse",
-    ):
+    ) -> None:
         """
         Initialize the ChineseDocumentSplitter component.
 
@@ -95,6 +95,9 @@ class ChineseDocumentSplitter:
         self.respect_sentence_boundary = respect_sentence_boundary
         self.splitting_function = splitting_function
         self.granularity = granularity
+        self.chinese_tokenizer: HanlpComponent | None = None
+        self.split_sent: HanlpComponent | None = None
+        self._is_warmed_up = False
 
     @staticmethod
     def _validate_init_parameters(
@@ -159,9 +162,8 @@ class ChineseDocumentSplitter:
 
         :raises RuntimeError: If the Chinese word segmentation model is not loaded.
         """
-        if self.split_sent is None:
-            msg = "The Chinese word segmentation model is not loaded. Please run 'warm_up()' before calling 'run()'."
-            raise RuntimeError(msg)
+        if not self._is_warmed_up:
+            self.warm_up()
 
         split_docs = []
         for doc in documents:
@@ -170,11 +172,15 @@ class ChineseDocumentSplitter:
 
     def warm_up(self) -> None:
         """Warm up the component by loading the necessary models."""
+        if self._is_warmed_up:
+            return
+
         if self.granularity == "coarse":
             self.chinese_tokenizer = hanlp.load(hanlp.pretrained.tok.COARSE_ELECTRA_SMALL_ZH)
         if self.granularity == "fine":
             self.chinese_tokenizer = hanlp.load(hanlp.pretrained.tok.FINE_ELECTRA_SMALL_ZH)
         self.split_sent = hanlp.load(hanlp.pretrained.eos.UD_CTB_EOS_MUL)
+        self._is_warmed_up = True
 
     def _split_by_character(self, doc: Document) -> list[Document]:
         """
@@ -188,7 +194,7 @@ class ChineseDocumentSplitter:
 
         split_at = _CHARACTER_SPLIT_BY_MAPPING[self.split_by]
 
-        units = self.chinese_tokenizer(doc.content)
+        units = self.chinese_tokenizer(doc.content)  # type: ignore[misc]
 
         for i in range(len(units) - 1):
             units[i] += split_at
@@ -211,7 +217,7 @@ class ChineseDocumentSplitter:
         :return: A list of split sentences.
         """
         # Split sentences
-        sentences = self.split_sent(text)
+        sentences = self.split_sent(text)  # type: ignore[misc]
 
         # Organize the format of segmented sentences
         results = []
@@ -261,9 +267,15 @@ class ChineseDocumentSplitter:
 
         for sentence_idx, sentence in enumerate(sentences):
             current_chunk.append(sentence)
-            chunk_word_count += len(self.chinese_tokenizer(sentence))
+            chunk_word_count += len(
+                self.chinese_tokenizer(sentence)  # type: ignore[misc]
+            )
             next_sentence_word_count = (
-                len(self.chinese_tokenizer(sentences[sentence_idx + 1])) if sentence_idx < len(sentences) - 1 else 0
+                len(
+                    self.chinese_tokenizer(sentences[sentence_idx + 1])  # type: ignore[misc]
+                )
+                if sentence_idx < len(sentences) - 1
+                else 0
             )
 
             # Number of words in the current chunk plus the next sentence is larger than the split_length,
@@ -478,7 +490,9 @@ class ChineseDocumentSplitter:
         num_words = 0
 
         for sent in reversed(sentences[1:]):
-            num_words += len(self.chinese_tokenizer(sent))
+            num_words += len(
+                self.chinese_tokenizer(sent)  # type: ignore[misc]
+            )
             # If the number of words is larger than the split_length then don't add any more sentences
             if num_words > split_length:
                 break

{hanlp_haystack-2.0.0 → hanlp_haystack-2.1.1}/tests/test_chinese_document_splitter.py

@@ -93,7 +93,6 @@ class TestChineseDocumentSplitter:
     @pytest.mark.integration
     def test_empty_list(self):
         splitter = ChineseDocumentSplitter()
-        splitter.warm_up()
         results = splitter.run(documents=[])
         assert results == {"documents": []}
 
@@ -101,7 +100,6 @@ class TestChineseDocumentSplitter:
     def test_empty_document(self):
         splitter = ChineseDocumentSplitter()
         documents = [Document(content="")]
-        splitter.warm_up()
         results = splitter.run(documents=documents)
         assert results == {"documents": []}
 
@@ -109,7 +107,6 @@ class TestChineseDocumentSplitter:
     def test_whitespace_only_document(self):
         splitter = ChineseDocumentSplitter()
         documents = [Document(content="  ")]
-        splitter.warm_up()
         results = splitter.run(documents=documents)
         assert len(results["documents"]) == 0
 
@@ -120,7 +117,6 @@ class TestChineseDocumentSplitter:
             Document(content="这是另一个测试文本。", meta={"name": "doc 1"}),
         ]
         splitter = ChineseDocumentSplitter(split_by="word", split_length=5, split_overlap=2)
-        splitter.warm_up()
         result = splitter.run(documents=documents)
         assert len(result["documents"]) == 2
         for doc, split_doc in zip(documents, result["documents"], strict=True):
@@ -133,7 +129,6 @@ class TestChineseDocumentSplitter:
             Document(content="这是第二个测试文本。"),
         ]
         splitter = ChineseDocumentSplitter(split_by="word", split_length=5, split_overlap=2)
-        splitter.warm_up()
         result = splitter.run(documents=documents)
         assert len(result["documents"]) == 2
         for doc, split_doc in zip(documents, result["documents"], strict=True):
@@ -142,7 +137,6 @@ class TestChineseDocumentSplitter:
     @pytest.mark.integration
     def test_split_by_word(self, sample_text):
         splitter = ChineseDocumentSplitter(split_by="word", granularity="coarse", split_length=5, split_overlap=0)
-        splitter.warm_up()
         result = splitter.run(documents=[Document(content=sample_text)])
         docs = result["documents"]
         assert all(isinstance(doc, Document) for doc in docs)
@@ -153,7 +147,6 @@ class TestChineseDocumentSplitter:
     @pytest.mark.integration
     def test_split_by_sentence(self, sample_text):
         splitter = ChineseDocumentSplitter(split_by="sentence", granularity="coarse", split_length=10, split_overlap=0)
-        splitter.warm_up()
         result = splitter.run(documents=[Document(content=sample_text)])
         docs = result["documents"]
         assert all(isinstance(doc, Document) for doc in docs), "All docs should be instances of Document"
@@ -170,7 +163,6 @@ class TestChineseDocumentSplitter:
         splitter = ChineseDocumentSplitter(
             split_by="word", split_length=10, split_overlap=3, respect_sentence_boundary=True
         )
-        splitter.warm_up()
         result = splitter.run(documents=[doc])
         docs = result["documents"]
 
@@ -193,7 +185,6 @@ class TestChineseDocumentSplitter:
         )
 
         splitter = ChineseDocumentSplitter(split_by="word", split_length=30, split_overlap=10, granularity="coarse")
-        splitter.warm_up()
         result = splitter.run(documents=[doc])
         docs = result["documents"]
         assert len(docs) == 6

hanlp_haystack-2.0.0/CHANGELOG.md

@@ -1,20 +0,0 @@
-# Changelog
-
-## [integrations/hanlp-v1.0.1] - 2025-10-14
-
-### 🐛 Bug Fixes
-
-- Missing decorator output for ChineseDocumentSplitter (#2377)
-
-
-### 🧹 Chores
-
-- Standardize readmes - part 2 (#2205)
-
-## [integrations/hanlp-v1.0.0] - 2025-07-22
-
-### 🚀 Features
-
-- Add new HanLP integration with ChineseDocumentSplitter (#1943)
-
-<!-- generated by git-cliff -->