hanlp-haystack 1.0.1__tar.gz → 2.1.0__tar.gz

hanlp_haystack-2.1.0/CHANGELOG.md

@@ -0,0 +1,44 @@
+# Changelog
+
+## [integrations/hanlp-v2.0.0] - 2026-01-12
+
+### 🐛 Bug Fixes
+
+- Docs: fix hanlp markdown filenames (#2480)
+
+### 📚 Documentation
+
+- Add pydoc configurations for Docusaurus (#2411)
+
+### ⚙️ CI
+
+- Change pytest command (#2475)
+
+### 🧹 Chores
+
+- Remove Readme API CI workflow and configs (#2573)
+- Make fmt command more forgiving (#2671)
+- [**breaking**] Hanlp - drop Python 3.9 and use X|Y typing (#2707)
+
+### 🌀 Miscellaneous
+
+- Enhancement: Adopt PEP 585 type hinting (part 4) (#2527)
+
+## [integrations/hanlp-v1.0.1] - 2025-10-14
+
+### 🐛 Bug Fixes
+
+- Missing decorator output for ChineseDocumentSplitter (#2377)
+
+
+### 🧹 Chores
+
+- Standardize readmes - part 2 (#2205)
+
+## [integrations/hanlp-v1.0.0] - 2025-07-22
+
+### 🚀 Features
+
+- Add new HanLP integration with ChineseDocumentSplitter (#1943)
+
+<!-- generated by git-cliff -->

{hanlp_haystack-1.0.1 → hanlp_haystack-2.1.0}/LICENSE.txt

@@ -58,7 +58,7 @@ APPENDIX: How to apply the Apache License to your work.
 
 To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!)  The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives.
 
-Copyright [yyyy] [name of copyright owner]
+Copyright 2025 deepset GmbH
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

{hanlp_haystack-1.0.1 → hanlp_haystack-2.1.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hanlp-haystack
-Version: 1.0.1
+Version: 2.1.0
 Summary: An integration of Han Language Processing - HanLP as a ChineseDocumentSplitter component.
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/hanlp#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -11,16 +11,15 @@ License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: hanlp>=2.1.1
-Requires-Dist: haystack-ai>=2.13.1
+Requires-Dist: haystack-ai>=2.22.0
 Description-Content-Type: text/markdown
 
 # hanlp-haystack

hanlp_haystack-2.1.0/pydoc/config_docusaurus.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_integrations.components.preprocessors.hanlp.chinese_document_splitter
+  search_path:
+  - ../src
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: HanLP integration for Haystack
+  id: integrations-hanlp
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: hanlp.md
+  title: HanLP
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

{hanlp_haystack-1.0.1 → hanlp_haystack-2.1.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = "hanlp-haystack"
 dynamic = ["version"]
 description = 'An integration of Han Language Processing - HanLP as a ChineseDocumentSplitter component.'
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = "Apache-2.0"
 keywords = []
 authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
@@ -15,7 +15,6 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
@@ -24,7 +23,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 dependencies = [
-  "haystack-ai>=2.13.1",
+  "haystack-ai>=2.22.0",
   "hanlp>=2.1.1"
 ]
 
@@ -49,8 +48,8 @@ installer = "uv"
 dependencies = ["haystack-pydoc-tools", "ruff"]
 
 [tool.hatch.envs.default.scripts]
-docs = ["pydoc-markdown pydoc/config.yml"]
-fmt = "ruff check --fix {args} && ruff format {args}"
+docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
+fmt = "ruff check --fix {args}; ruff format {args}"
 fmt-check = "ruff check {args} && ruff format --check {args}"
 
 [tool.hatch.envs.test]
@@ -67,7 +66,7 @@ dependencies = [
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
 all = 'pytest {args:tests}'
-cov-retry = 'all --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x'
+cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
 types = """mypy -p haystack_integrations.components.preprocessors.hanlp {args}"""
 
 [tool.mypy]
@@ -83,12 +82,11 @@ module = [
 ignore_missing_imports = true
 
 [tool.black]
-target-version = ["py38"]
+target-version = ["py39"]
 line-length = 120
 skip-string-normalization = true
 
 [tool.ruff]
-target-version = "py38"
 line-length = 120
 
 [tool.ruff.lint]
@@ -138,10 +136,6 @@ ignore = [
   "RUF001",
   "RUF002",
 ]
-unfixable = [
-  # Don't touch unused imports
-  "F401",
-]
 
 [tool.ruff.lint.isort]
 known-first-party = ["haystack_integrations"]

{hanlp_haystack-1.0.1 → hanlp_haystack-2.1.0}/src/haystack_integrations/components/preprocessors/hanlp/__init__.py

@@ -1,6 +1,7 @@
 # SPDX-FileCopyrightText: 2025-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
+
 from .chinese_document_splitter import ChineseDocumentSplitter
 
 __all__ = [

{hanlp_haystack-1.0.1 → hanlp_haystack-2.1.0}/src/haystack_integrations/components/preprocessors/hanlp/chinese_document_splitter.py

@@ -2,8 +2,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from collections.abc import Callable
 from copy import deepcopy
-from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
+from typing import Any, Literal
 
 from haystack import Document, component, logging
 from haystack.core.serialization import default_from_dict, default_to_dict
@@ -11,6 +12,7 @@ from haystack.utils import deserialize_callable, serialize_callable
 from more_itertools import windowed
 
 import hanlp
+from hanlp.common.component import Component as HanlpComponent
 
 logger = logging.getLogger(__name__)
 
@@ -46,7 +48,6 @@ class ChineseDocumentSplitter:
     splitter = ChineseDocumentSplitter(
         split_by="word", split_length=10, split_overlap=3, respect_sentence_boundary=True
     )
-    splitter.warm_up()
     result = splitter.run(documents=[doc])
     print(result["documents"])
     ```
@@ -59,9 +60,9 @@ class ChineseDocumentSplitter:
         split_overlap: int = 200,
         split_threshold: int = 0,
         respect_sentence_boundary: bool = False,
-        splitting_function: Optional[Callable] = None,
+        splitting_function: Callable | None = None,
         granularity: Literal["coarse", "fine"] = "coarse",
-    ):
+    ) -> None:
         """
         Initialize the ChineseDocumentSplitter component.
 
@@ -94,6 +95,9 @@ class ChineseDocumentSplitter:
         self.respect_sentence_boundary = respect_sentence_boundary
         self.splitting_function = splitting_function
         self.granularity = granularity
+        self.chinese_tokenizer: HanlpComponent | None = None
+        self.split_sent: HanlpComponent | None = None
+        self._is_warmed_up = False
 
     @staticmethod
     def _validate_init_parameters(
@@ -149,7 +153,7 @@ class ChineseDocumentSplitter:
             raise ValueError(msg)
 
     @component.output_types(documents=list[Document])
-    def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
         """
         Split documents into smaller chunks.
 
@@ -158,9 +162,8 @@ class ChineseDocumentSplitter:
 
         :raises RuntimeError: If the Chinese word segmentation model is not loaded.
         """
-        if self.split_sent is None:
-            msg = "The Chinese word segmentation model is not loaded. Please run 'warm_up()' before calling 'run()'."
-            raise RuntimeError(msg)
+        if not self._is_warmed_up:
+            self.warm_up()
 
         split_docs = []
         for doc in documents:
@@ -169,13 +172,17 @@ class ChineseDocumentSplitter:
 
     def warm_up(self) -> None:
         """Warm up the component by loading the necessary models."""
+        if self._is_warmed_up:
+            return
+
         if self.granularity == "coarse":
             self.chinese_tokenizer = hanlp.load(hanlp.pretrained.tok.COARSE_ELECTRA_SMALL_ZH)
         if self.granularity == "fine":
             self.chinese_tokenizer = hanlp.load(hanlp.pretrained.tok.FINE_ELECTRA_SMALL_ZH)
         self.split_sent = hanlp.load(hanlp.pretrained.eos.UD_CTB_EOS_MUL)
+        self._is_warmed_up = True
 
-    def _split_by_character(self, doc: Document) -> List[Document]:
+    def _split_by_character(self, doc: Document) -> list[Document]:
         """
         Define a function to handle Chinese clauses
 
@@ -187,7 +194,7 @@ class ChineseDocumentSplitter:
 
         split_at = _CHARACTER_SPLIT_BY_MAPPING[self.split_by]
 
-        units = self.chinese_tokenizer(doc.content)
+        units = self.chinese_tokenizer(doc.content)  # type: ignore[misc]
 
         for i in range(len(units) - 1):
             units[i] += split_at
@@ -202,7 +209,7 @@ class ChineseDocumentSplitter:
         )
 
     # Define a function to handle Chinese clauses
-    def chinese_sentence_split(self, text: str) -> List[Dict[str, Any]]:
+    def chinese_sentence_split(self, text: str) -> list[dict[str, Any]]:
         """
         Split Chinese text into sentences.
 
@@ -210,7 +217,7 @@ class ChineseDocumentSplitter:
         :return: A list of split sentences.
         """
         # Split sentences
-        sentences = self.split_sent(text)
+        sentences = self.split_sent(text)  # type: ignore[misc]
 
         # Organize the format of segmented sentences
         results = []
@@ -223,7 +230,7 @@ class ChineseDocumentSplitter:
 
         return results
 
-    def _split_document(self, doc: Document) -> List[Document]:
+    def _split_document(self, doc: Document) -> list[Document]:
         if self.split_by == "sentence" or self.respect_sentence_boundary:
             return self._split_by_hanlp_sentence(doc)
 
@@ -233,8 +240,8 @@ class ChineseDocumentSplitter:
         return self._split_by_character(doc)
 
     def _concatenate_sentences_based_on_word_amount(
-        self, sentences: List[str], split_length: int, split_overlap: int, granularity: str
-    ) -> Tuple[List[str], List[int], List[int]]:
+        self, sentences: list[str], split_length: int, split_overlap: int, granularity: str
+    ) -> tuple[list[str], list[int], list[int]]:
         """
         Groups the sentences into chunks of `split_length` words while respecting sentence boundaries.
 
@@ -251,18 +258,24 @@ class ChineseDocumentSplitter:
         chunk_word_count = 0
         chunk_starting_page_number = 1
         chunk_start_idx = 0
-        current_chunk: List[str] = []
+        current_chunk: list[str] = []
 
         # output lists
         split_start_page_numbers = []
-        list_of_splits: List[List[str]] = []
+        list_of_splits: list[list[str]] = []
         split_start_indices = []
 
         for sentence_idx, sentence in enumerate(sentences):
             current_chunk.append(sentence)
-            chunk_word_count += len(self.chinese_tokenizer(sentence))
+            chunk_word_count += len(
+                self.chinese_tokenizer(sentence)  # type: ignore[misc]
+            )
             next_sentence_word_count = (
-                len(self.chinese_tokenizer(sentences[sentence_idx + 1])) if sentence_idx < len(sentences) - 1 else 0
+                len(
+                    self.chinese_tokenizer(sentences[sentence_idx + 1])  # type: ignore[misc]
+                )
+                if sentence_idx < len(sentences) - 1
+                else 0
             )
 
             # Number of words in the current chunk plus the next sentence is larger than the split_length,
@@ -305,7 +318,7 @@ class ChineseDocumentSplitter:
 
         return text_splits, split_start_page_numbers, split_start_indices
 
-    def _split_by_hanlp_sentence(self, doc: Document) -> List[Document]:
+    def _split_by_hanlp_sentence(self, doc: Document) -> list[Document]:
         """
         Split Chinese text into sentences.
 
@@ -342,8 +355,8 @@ class ChineseDocumentSplitter:
         return split_docs
 
     def _concatenate_units(
-        self, elements: List[str], split_length: int, split_overlap: int, split_threshold: int
-    ) -> Tuple[List[str], List[int], List[int]]:
+        self, elements: list[str], split_length: int, split_overlap: int, split_threshold: int
+    ) -> tuple[list[str], list[int], list[int]]:
         """
         Concatenates the elements into parts of split_length units.
 
@@ -364,9 +377,9 @@ class ChineseDocumentSplitter:
         # Otherwise, proceed as before
         step = split_length - split_overlap
         step = max(step, 1)
-        text_splits: List[str] = []
-        splits_pages: List[int] = []
-        splits_start_idxs: List[int] = []
+        text_splits: list[str] = []
+        splits_pages: list[int] = []
+        splits_start_idxs: list[int] = []
         cur_start_idx = 0
         cur_page = 1
         segments = windowed(elements, n=split_length, step=step)
@@ -399,14 +412,14 @@ class ChineseDocumentSplitter:
         return text_splits, splits_pages, splits_start_idxs
 
     def _create_docs_from_splits(
-        self, text_splits: List[str], splits_pages: List[int], splits_start_idxs: List[int], meta: Dict[str, Any]
-    ) -> List[Document]:
+        self, text_splits: list[str], splits_pages: list[int], splits_start_idxs: list[int], meta: dict[str, Any]
+    ) -> list[Document]:
         """
         Creates Document objects from splits enriching them with page number and the metadata of the original document.
         """
-        documents: List[Document] = []
+        documents: list[Document] = []
 
-        for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs)):
+        for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs, strict=True)):
             copied_meta = deepcopy(meta)
             copied_meta["page_number"] = splits_pages[i]
             copied_meta["split_id"] = i
@@ -460,7 +473,7 @@ class ChineseDocumentSplitter:
                 overlapping_range = (0, overlapping_range[1] - overlapping_range[0])
                 previous_doc.meta["_split_overlap"].append({"doc_id": current_doc.id, "range": overlapping_range})
 
-    def _number_of_sentences_to_keep(self, sentences: List[str], split_length: int, split_overlap: int) -> int:
+    def _number_of_sentences_to_keep(self, sentences: list[str], split_length: int, split_overlap: int) -> int:
         """
         Returns the number of sentences to keep in the next chunk based on the `split_overlap` and `split_length`.
 
@@ -477,7 +490,9 @@ class ChineseDocumentSplitter:
         num_words = 0
 
         for sent in reversed(sentences[1:]):
-            num_words += len(self.chinese_tokenizer(sent))
+            num_words += len(
+                self.chinese_tokenizer(sent)  # type: ignore[misc]
+            )
             # If the number of words is larger than the split_length then don't add any more sentences
             if num_words > split_length:
                 break
@@ -486,7 +501,7 @@ class ChineseDocumentSplitter:
                 break
         return num_sentences_to_keep
 
-    def _split_by_function(self, doc: Document) -> List[Document]:
+    def _split_by_function(self, doc: Document) -> list[Document]:
         """
         Split a document using a custom splitting function.
 
@@ -515,7 +530,7 @@ class ChineseDocumentSplitter:
             meta=metadata,
         )
 
-    def to_dict(self) -> Dict[str, Any]:
+    def to_dict(self) -> dict[str, Any]:
         """
         Serializes the component to a dictionary.
         """
@@ -533,7 +548,7 @@ class ChineseDocumentSplitter:
         return serialized
 
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "ChineseDocumentSplitter":
+    def from_dict(cls, data: dict[str, Any]) -> "ChineseDocumentSplitter":
         """
         Deserializes the component from a dictionary.
         """

{hanlp_haystack-1.0.1 → hanlp_haystack-2.1.0}/tests/test_chinese_document_splitter.py

@@ -93,7 +93,6 @@ class TestChineseDocumentSplitter:
     @pytest.mark.integration
     def test_empty_list(self):
         splitter = ChineseDocumentSplitter()
-        splitter.warm_up()
         results = splitter.run(documents=[])
         assert results == {"documents": []}
 
@@ -101,7 +100,6 @@ class TestChineseDocumentSplitter:
     def test_empty_document(self):
         splitter = ChineseDocumentSplitter()
         documents = [Document(content="")]
-        splitter.warm_up()
         results = splitter.run(documents=documents)
         assert results == {"documents": []}
 
@@ -109,7 +107,6 @@ class TestChineseDocumentSplitter:
     def test_whitespace_only_document(self):
         splitter = ChineseDocumentSplitter()
         documents = [Document(content="  ")]
-        splitter.warm_up()
         results = splitter.run(documents=documents)
         assert len(results["documents"]) == 0
 
@@ -120,10 +117,9 @@ class TestChineseDocumentSplitter:
             Document(content="这是另一个测试文本。", meta={"name": "doc 1"}),
         ]
         splitter = ChineseDocumentSplitter(split_by="word", split_length=5, split_overlap=2)
-        splitter.warm_up()
         result = splitter.run(documents=documents)
         assert len(result["documents"]) == 2
-        for doc, split_doc in zip(documents, result["documents"]):
+        for doc, split_doc in zip(documents, result["documents"], strict=True):
             assert doc.meta.items() <= split_doc.meta.items()
 
     @pytest.mark.integration
@@ -133,16 +129,14 @@ class TestChineseDocumentSplitter:
             Document(content="这是第二个测试文本。"),
         ]
         splitter = ChineseDocumentSplitter(split_by="word", split_length=5, split_overlap=2)
-        splitter.warm_up()
         result = splitter.run(documents=documents)
         assert len(result["documents"]) == 2
-        for doc, split_doc in zip(documents, result["documents"]):
+        for doc, split_doc in zip(documents, result["documents"], strict=True):
             assert doc.id == split_doc.meta["source_id"]
 
     @pytest.mark.integration
     def test_split_by_word(self, sample_text):
         splitter = ChineseDocumentSplitter(split_by="word", granularity="coarse", split_length=5, split_overlap=0)
-        splitter.warm_up()
         result = splitter.run(documents=[Document(content=sample_text)])
         docs = result["documents"]
         assert all(isinstance(doc, Document) for doc in docs)
@@ -153,7 +147,6 @@ class TestChineseDocumentSplitter:
     @pytest.mark.integration
     def test_split_by_sentence(self, sample_text):
         splitter = ChineseDocumentSplitter(split_by="sentence", granularity="coarse", split_length=10, split_overlap=0)
-        splitter.warm_up()
         result = splitter.run(documents=[Document(content=sample_text)])
         docs = result["documents"]
         assert all(isinstance(doc, Document) for doc in docs), "All docs should be instances of Document"
@@ -170,7 +163,6 @@ class TestChineseDocumentSplitter:
         splitter = ChineseDocumentSplitter(
             split_by="word", split_length=10, split_overlap=3, respect_sentence_boundary=True
         )
-        splitter.warm_up()
         result = splitter.run(documents=[doc])
         docs = result["documents"]
 
@@ -193,7 +185,6 @@ class TestChineseDocumentSplitter:
         )
 
         splitter = ChineseDocumentSplitter(split_by="word", split_length=30, split_overlap=10, granularity="coarse")
-        splitter.warm_up()
         result = splitter.run(documents=[doc])
         docs = result["documents"]
         assert len(docs) == 6

hanlp_haystack-1.0.1/CHANGELOG.md

@@ -1,9 +0,0 @@
-# Changelog
-
-## [integrations/hanlp-v1.0.0] - 2025-07-22
-
-### 🚀 Features
-
-- Add new HanLP integration with ChineseDocumentSplitter (#1943)
-
-<!-- generated by git-cliff -->

hanlp_haystack-1.0.1/pydoc/config.yml

@@ -1,29 +0,0 @@
-loaders:
-  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
-    search_path: [../src]
-    modules: [
-      "haystack_integrations.components.preprocessors.hanlp.chinese_document_splitter",
-    ]
-    ignore_when_discovered: ["__init__"]
-processors:
-  - type: filter
-    expression:
-    documented_only: true
-    do_not_filter_modules: false
-    skip_empty_modules: true
-  - type: smart
-  - type: crossref
-renderer:
-  type: haystack_pydoc_tools.renderers.ReadmeIntegrationRenderer
-  excerpt: HanLP integration for Haystack
-  category_slug: integrations-api
-  title: HanLP
-  slug: integrations-hanlp
-  order: 100
-  markdown:
-    descriptive_class_title: false
-    classdef_code_block: false
-    descriptive_module_title: true
-    add_method_class_prefix: true
-    add_member_class_prefix: false
-    filename: _readme_github.md