hanlp-haystack 1.0.1__tar.gz → 2.0.0__tar.gz

hanlp_haystack-2.0.0/CHANGELOG.md

@@ -0,0 +1,20 @@
+# Changelog
+
+## [integrations/hanlp-v1.0.1] - 2025-10-14
+
+### 🐛 Bug Fixes
+
+- Missing decorator output for ChineseDocumentSplitter (#2377)
+
+
+### 🧹 Chores
+
+- Standardize readmes - part 2 (#2205)
+
+## [integrations/hanlp-v1.0.0] - 2025-07-22
+
+### 🚀 Features
+
+- Add new HanLP integration with ChineseDocumentSplitter (#1943)
+
+<!-- generated by git-cliff -->

{hanlp_haystack-1.0.1 → hanlp_haystack-2.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: hanlp-haystack
-Version: 1.0.1
+Version: 2.0.0
 Summary: An integration of Han Language Processing - HanLP as a ChineseDocumentSplitter component.
 Project-URL: Documentation, https://github.com/deepset-ai/haystack-core-integrations/tree/main/integrations/hanlp#readme
 Project-URL: Issues, https://github.com/deepset-ai/haystack-core-integrations/issues
@@ -11,16 +11,15 @@ License-File: LICENSE.txt
 Classifier: Development Status :: 4 - Beta
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: Implementation :: CPython
 Classifier: Programming Language :: Python :: Implementation :: PyPy
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Requires-Dist: hanlp>=2.1.1
-Requires-Dist: haystack-ai>=2.13.1
+Requires-Dist: haystack-ai>=2.22.0
 Description-Content-Type: text/markdown
 
 # hanlp-haystack

hanlp_haystack-2.0.0/pydoc/config_docusaurus.yml

@@ -0,0 +1,28 @@
+loaders:
+- ignore_when_discovered:
+  - __init__
+  modules:
+  - haystack_integrations.components.preprocessors.hanlp.chinese_document_splitter
+  search_path:
+  - ../src
+  type: haystack_pydoc_tools.loaders.CustomPythonLoader
+processors:
+- do_not_filter_modules: false
+  documented_only: true
+  expression: null
+  skip_empty_modules: true
+  type: filter
+- type: smart
+- type: crossref
+renderer:
+  description: HanLP integration for Haystack
+  id: integrations-hanlp
+  markdown:
+    add_member_class_prefix: false
+    add_method_class_prefix: true
+    classdef_code_block: false
+    descriptive_class_title: false
+    descriptive_module_title: true
+    filename: hanlp.md
+  title: HanLP
+  type: haystack_pydoc_tools.renderers.DocusaurusRenderer

{hanlp_haystack-1.0.1 → hanlp_haystack-2.0.0}/pyproject.toml

@@ -7,7 +7,7 @@ name = "hanlp-haystack"
 dynamic = ["version"]
 description = 'An integration of Han Language Processing - HanLP as a ChineseDocumentSplitter component.'
 readme = "README.md"
-requires-python = ">=3.9"
+requires-python = ">=3.10"
 license = "Apache-2.0"
 keywords = []
 authors = [{ name = "deepset GmbH", email = "info@deepset.ai" }]
@@ -15,7 +15,6 @@ classifiers = [
   "License :: OSI Approved :: Apache Software License",
   "Development Status :: 4 - Beta",
   "Programming Language :: Python",
-  "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
@@ -24,7 +23,7 @@ classifiers = [
   "Programming Language :: Python :: Implementation :: PyPy",
 ]
 dependencies = [
-  "haystack-ai>=2.13.1",
+  "haystack-ai>=2.22.0",
   "hanlp>=2.1.1"
 ]
 
@@ -49,8 +48,8 @@ installer = "uv"
 dependencies = ["haystack-pydoc-tools", "ruff"]
 
 [tool.hatch.envs.default.scripts]
-docs = ["pydoc-markdown pydoc/config.yml"]
-fmt = "ruff check --fix {args} && ruff format {args}"
+docs = ["pydoc-markdown pydoc/config_docusaurus.yml"]
+fmt = "ruff check --fix {args}; ruff format {args}"
 fmt-check = "ruff check {args} && ruff format --check {args}"
 
 [tool.hatch.envs.test]
@@ -67,7 +66,7 @@ dependencies = [
 unit = 'pytest -m "not integration" {args:tests}'
 integration = 'pytest -m "integration" {args:tests}'
 all = 'pytest {args:tests}'
-cov-retry = 'all --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x'
+cov-retry = 'pytest --cov=haystack_integrations --reruns 3 --reruns-delay 30 -x {args:tests}'
 types = """mypy -p haystack_integrations.components.preprocessors.hanlp {args}"""
 
 [tool.mypy]
@@ -83,12 +82,11 @@ module = [
 ignore_missing_imports = true
 
 [tool.black]
-target-version = ["py38"]
+target-version = ["py39"]
 line-length = 120
 skip-string-normalization = true
 
 [tool.ruff]
-target-version = "py38"
 line-length = 120
 
 [tool.ruff.lint]
@@ -138,10 +136,6 @@ ignore = [
   "RUF001",
   "RUF002",
 ]
-unfixable = [
-  # Don't touch unused imports
-  "F401",
-]
 
 [tool.ruff.lint.isort]
 known-first-party = ["haystack_integrations"]

{hanlp_haystack-1.0.1 → hanlp_haystack-2.0.0}/src/haystack_integrations/components/preprocessors/hanlp/chinese_document_splitter.py

@@ -2,8 +2,9 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
+from collections.abc import Callable
 from copy import deepcopy
-from typing import Any, Callable, Dict, List, Literal, Optional, Tuple
+from typing import Any, Literal
 
 from haystack import Document, component, logging
 from haystack.core.serialization import default_from_dict, default_to_dict
@@ -59,7 +60,7 @@ class ChineseDocumentSplitter:
         split_overlap: int = 200,
         split_threshold: int = 0,
         respect_sentence_boundary: bool = False,
-        splitting_function: Optional[Callable] = None,
+        splitting_function: Callable | None = None,
         granularity: Literal["coarse", "fine"] = "coarse",
     ):
         """
@@ -149,7 +150,7 @@ class ChineseDocumentSplitter:
             raise ValueError(msg)
 
     @component.output_types(documents=list[Document])
-    def run(self, documents: List[Document]) -> Dict[str, List[Document]]:
+    def run(self, documents: list[Document]) -> dict[str, list[Document]]:
         """
         Split documents into smaller chunks.
 
@@ -175,7 +176,7 @@ class ChineseDocumentSplitter:
             self.chinese_tokenizer = hanlp.load(hanlp.pretrained.tok.FINE_ELECTRA_SMALL_ZH)
         self.split_sent = hanlp.load(hanlp.pretrained.eos.UD_CTB_EOS_MUL)
 
-    def _split_by_character(self, doc: Document) -> List[Document]:
+    def _split_by_character(self, doc: Document) -> list[Document]:
         """
         Define a function to handle Chinese clauses
 
@@ -202,7 +203,7 @@ class ChineseDocumentSplitter:
         )
 
     # Define a function to handle Chinese clauses
-    def chinese_sentence_split(self, text: str) -> List[Dict[str, Any]]:
+    def chinese_sentence_split(self, text: str) -> list[dict[str, Any]]:
         """
         Split Chinese text into sentences.
 
@@ -223,7 +224,7 @@ class ChineseDocumentSplitter:
 
         return results
 
-    def _split_document(self, doc: Document) -> List[Document]:
+    def _split_document(self, doc: Document) -> list[Document]:
         if self.split_by == "sentence" or self.respect_sentence_boundary:
             return self._split_by_hanlp_sentence(doc)
 
@@ -233,8 +234,8 @@ class ChineseDocumentSplitter:
         return self._split_by_character(doc)
 
     def _concatenate_sentences_based_on_word_amount(
-        self, sentences: List[str], split_length: int, split_overlap: int, granularity: str
-    ) -> Tuple[List[str], List[int], List[int]]:
+        self, sentences: list[str], split_length: int, split_overlap: int, granularity: str
+    ) -> tuple[list[str], list[int], list[int]]:
         """
         Groups the sentences into chunks of `split_length` words while respecting sentence boundaries.
 
@@ -251,11 +252,11 @@ class ChineseDocumentSplitter:
         chunk_word_count = 0
         chunk_starting_page_number = 1
         chunk_start_idx = 0
-        current_chunk: List[str] = []
+        current_chunk: list[str] = []
 
         # output lists
         split_start_page_numbers = []
-        list_of_splits: List[List[str]] = []
+        list_of_splits: list[list[str]] = []
         split_start_indices = []
 
         for sentence_idx, sentence in enumerate(sentences):
@@ -305,7 +306,7 @@ class ChineseDocumentSplitter:
 
         return text_splits, split_start_page_numbers, split_start_indices
 
-    def _split_by_hanlp_sentence(self, doc: Document) -> List[Document]:
+    def _split_by_hanlp_sentence(self, doc: Document) -> list[Document]:
         """
         Split Chinese text into sentences.
 
@@ -342,8 +343,8 @@ class ChineseDocumentSplitter:
         return split_docs
 
     def _concatenate_units(
-        self, elements: List[str], split_length: int, split_overlap: int, split_threshold: int
-    ) -> Tuple[List[str], List[int], List[int]]:
+        self, elements: list[str], split_length: int, split_overlap: int, split_threshold: int
+    ) -> tuple[list[str], list[int], list[int]]:
         """
         Concatenates the elements into parts of split_length units.
 
@@ -364,9 +365,9 @@ class ChineseDocumentSplitter:
         # Otherwise, proceed as before
         step = split_length - split_overlap
         step = max(step, 1)
-        text_splits: List[str] = []
-        splits_pages: List[int] = []
-        splits_start_idxs: List[int] = []
+        text_splits: list[str] = []
+        splits_pages: list[int] = []
+        splits_start_idxs: list[int] = []
         cur_start_idx = 0
         cur_page = 1
         segments = windowed(elements, n=split_length, step=step)
@@ -399,14 +400,14 @@ class ChineseDocumentSplitter:
         return text_splits, splits_pages, splits_start_idxs
 
     def _create_docs_from_splits(
-        self, text_splits: List[str], splits_pages: List[int], splits_start_idxs: List[int], meta: Dict[str, Any]
-    ) -> List[Document]:
+        self, text_splits: list[str], splits_pages: list[int], splits_start_idxs: list[int], meta: dict[str, Any]
+    ) -> list[Document]:
         """
         Creates Document objects from splits enriching them with page number and the metadata of the original document.
         """
-        documents: List[Document] = []
+        documents: list[Document] = []
 
-        for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs)):
+        for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs, strict=True)):
             copied_meta = deepcopy(meta)
             copied_meta["page_number"] = splits_pages[i]
             copied_meta["split_id"] = i
@@ -460,7 +461,7 @@ class ChineseDocumentSplitter:
                 overlapping_range = (0, overlapping_range[1] - overlapping_range[0])
                 previous_doc.meta["_split_overlap"].append({"doc_id": current_doc.id, "range": overlapping_range})
 
-    def _number_of_sentences_to_keep(self, sentences: List[str], split_length: int, split_overlap: int) -> int:
+    def _number_of_sentences_to_keep(self, sentences: list[str], split_length: int, split_overlap: int) -> int:
         """
         Returns the number of sentences to keep in the next chunk based on the `split_overlap` and `split_length`.
 
@@ -486,7 +487,7 @@ class ChineseDocumentSplitter:
                 break
         return num_sentences_to_keep
 
-    def _split_by_function(self, doc: Document) -> List[Document]:
+    def _split_by_function(self, doc: Document) -> list[Document]:
         """
         Split a document using a custom splitting function.
 
@@ -515,7 +516,7 @@ class ChineseDocumentSplitter:
             meta=metadata,
         )
 
-    def to_dict(self) -> Dict[str, Any]:
+    def to_dict(self) -> dict[str, Any]:
         """
         Serializes the component to a dictionary.
         """
@@ -533,7 +534,7 @@ class ChineseDocumentSplitter:
         return serialized
 
     @classmethod
-    def from_dict(cls, data: Dict[str, Any]) -> "ChineseDocumentSplitter":
+    def from_dict(cls, data: dict[str, Any]) -> "ChineseDocumentSplitter":
         """
         Deserializes the component from a dictionary.
         """

{hanlp_haystack-1.0.1 → hanlp_haystack-2.0.0}/tests/test_chinese_document_splitter.py

@@ -123,7 +123,7 @@ class TestChineseDocumentSplitter:
         splitter.warm_up()
         result = splitter.run(documents=documents)
         assert len(result["documents"]) == 2
-        for doc, split_doc in zip(documents, result["documents"]):
+        for doc, split_doc in zip(documents, result["documents"], strict=True):
             assert doc.meta.items() <= split_doc.meta.items()
 
     @pytest.mark.integration
@@ -136,7 +136,7 @@ class TestChineseDocumentSplitter:
         splitter.warm_up()
         result = splitter.run(documents=documents)
         assert len(result["documents"]) == 2
-        for doc, split_doc in zip(documents, result["documents"]):
+        for doc, split_doc in zip(documents, result["documents"], strict=True):
             assert doc.id == split_doc.meta["source_id"]
 
     @pytest.mark.integration

hanlp_haystack-1.0.1/CHANGELOG.md

@@ -1,9 +0,0 @@
-# Changelog
-
-## [integrations/hanlp-v1.0.0] - 2025-07-22
-
-### 🚀 Features
-
-- Add new HanLP integration with ChineseDocumentSplitter (#1943)
-
-<!-- generated by git-cliff -->

hanlp_haystack-1.0.1/pydoc/config.yml

@@ -1,29 +0,0 @@
-loaders:
-  - type: haystack_pydoc_tools.loaders.CustomPythonLoader
-    search_path: [../src]
-    modules: [
-      "haystack_integrations.components.preprocessors.hanlp.chinese_document_splitter",
-    ]
-    ignore_when_discovered: ["__init__"]
-processors:
-  - type: filter
-    expression:
-    documented_only: true
-    do_not_filter_modules: false
-    skip_empty_modules: true
-  - type: smart
-  - type: crossref
-renderer:
-  type: haystack_pydoc_tools.renderers.ReadmeIntegrationRenderer
-  excerpt: HanLP integration for Haystack
-  category_slug: integrations-api
-  title: HanLP
-  slug: integrations-hanlp
-  order: 100
-  markdown:
-    descriptive_class_title: false
-    classdef_code_block: false
-    descriptive_module_title: true
-    add_method_class_prefix: true
-    add_member_class_prefix: false
-    filename: _readme_github.md