omni-split 0.0.1rc0__tar.gz

omni_split-0.0.1rc0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 M5Stack Technology CO LTD
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

omni_split-0.0.1rc0/PKG-INFO

@@ -0,0 +1,139 @@
+Metadata-Version: 2.4
+Name: omni_split
+Version: 0.0.1rc0
+Summary: A comprehensive document splitting toolkit
+Home-page: https://github.com/dinobot22/omni_split
+Author: dinobot22
+Author-email: 2802701695yyb@gmail.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: mistletoe
+Requires-Dist: transformers
+Requires-Dist: markitdown[docx,pptx,xls,xlsx]
+Requires-Dist: python-docx
+Requires-Dist: loguru
+Requires-Dist: wand
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+
+
+
+# omni_split: Split commonly used document (md, doc etc.) forms for RAG that support LLM.
+---
+<img src="./docs/icon.png" alt="omni_split" >
+
+---
+###  note: All other text formats are highly recommended to be converted to Markdown, and we focus on optimizing documents for Markdown. 
+---
+# usage
+###  install 
+```bash
+pip install omni_split
+```
+###  use case
+```python 
+import json
+from omni_split import OmniSplit
+from omni_split import word_preprocessing_and_return_bytesIO
+from omni_split import download_files_to_test_doc
+
+### == step 2: download test_doc file ==
+
+doc_dict = download_files_to_test_doc()
+text_doc_file_path = doc_dict["text_test.txt"]
+json_list_doc_file_path = doc_dict["json_list_test.json"]
+markdown_doc_file_path = doc_dict["markdown_test.md"]
+word_doc_file_path = doc_dict["docx_test.docx"]
+
+
+### == step 3: split  to chunk ==
+
+omni_spliter = OmniSplit()
+
+## note: test text split
+test_text = True
+if test_text:
+    with open(text_doc_file_path, "r") as f:
+        text_content = "".join(f.readlines())
+    res = omni_spliter.text_chunk_func(text_content,txt_chunk_size=1000)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+## note: test markdown json split
+test_markdown = True
+if test_markdown:
+    with open(json_list_doc_file_path, "r") as f:
+        md_content_json = json.load(f)
+    res = omni_spliter.markdown_json_chunk_func(md_content_json)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.markdown_json_chunk_func(md_content_json, clear_model=True)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+## note: test markdown split
+test_markdown = True
+if test_markdown:
+    with open(markdown_doc_file_path, "r") as f:
+        md_content = f.read()
+    res = omni_spliter.markdown_chunk_func(md_content)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.markdown_chunk_func(md_content, clear_model=True)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+
+## note: test word split
+test_document = True
+if test_document:
+
+    new_doc_io = word_preprocessing_and_return_bytesIO(word_doc_file_path)
+    res = omni_spliter.document_chunk_func(new_doc_io, txt_chunk_size=1000, clear_model=False)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.document_chunk_func(new_doc_io, txt_chunk_size=1000, clear_model=False, save_local_images_dir="./images")
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.document_chunk_func(new_doc_io, txt_chunk_size=1000, clear_model=True)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+```
+# Reminder of dependency:
+To automatically convert binary metafiles(e.g. x-wmf.) in Word to PNG, you need to install ImageMagick on Linux. 
+Try to install:
+https://docs.wand-py.org/en/latest/guide/install.html

omni_split-0.0.1rc0/README.md

@@ -0,0 +1,109 @@
+
+
+
+# omni_split: Split commonly used document (md, doc etc.) forms for RAG that support LLM.
+---
+<img src="./docs/icon.png" alt="omni_split" >
+
+---
+###  note: All other text formats are highly recommended to be converted to Markdown, and we focus on optimizing documents for Markdown. 
+---
+# usage
+###  install 
+```bash
+pip install omni_split
+```
+###  use case
+```python 
+import json
+from omni_split import OmniSplit
+from omni_split import word_preprocessing_and_return_bytesIO
+from omni_split import download_files_to_test_doc
+
+### == step 2: download test_doc file ==
+
+doc_dict = download_files_to_test_doc()
+text_doc_file_path = doc_dict["text_test.txt"]
+json_list_doc_file_path = doc_dict["json_list_test.json"]
+markdown_doc_file_path = doc_dict["markdown_test.md"]
+word_doc_file_path = doc_dict["docx_test.docx"]
+
+
+### == step 3: split  to chunk ==
+
+omni_spliter = OmniSplit()
+
+## note: test text split
+test_text = True
+if test_text:
+    with open(text_doc_file_path, "r") as f:
+        text_content = "".join(f.readlines())
+    res = omni_spliter.text_chunk_func(text_content,txt_chunk_size=1000)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+## note: test markdown json split
+test_markdown = True
+if test_markdown:
+    with open(json_list_doc_file_path, "r") as f:
+        md_content_json = json.load(f)
+    res = omni_spliter.markdown_json_chunk_func(md_content_json)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.markdown_json_chunk_func(md_content_json, clear_model=True)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+## note: test markdown split
+test_markdown = True
+if test_markdown:
+    with open(markdown_doc_file_path, "r") as f:
+        md_content = f.read()
+    res = omni_spliter.markdown_chunk_func(md_content)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.markdown_chunk_func(md_content, clear_model=True)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+
+## note: test word split
+test_document = True
+if test_document:
+
+    new_doc_io = word_preprocessing_and_return_bytesIO(word_doc_file_path)
+    res = omni_spliter.document_chunk_func(new_doc_io, txt_chunk_size=1000, clear_model=False)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.document_chunk_func(new_doc_io, txt_chunk_size=1000, clear_model=False, save_local_images_dir="./images")
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+    res = omni_spliter.document_chunk_func(new_doc_io, txt_chunk_size=1000, clear_model=True)
+    for item in res:
+        print(item)
+        print("------------")
+    print("=" * 10)
+
+```
+# Reminder of dependency:
+To automatically convert binary metafiles(e.g. x-wmf.) in Word to PNG, you need to install ImageMagick on Linux. 
+Try to install:
+https://docs.wand-py.org/en/latest/guide/install.html

omni_split-0.0.1rc0/omni_split/__init__.py

@@ -0,0 +1,16 @@
+from .omni_split import OmniSplit
+from .utils.base_utils import word_preprocessing_and_return_bytesIO
+from .utils.download_test_doc import download_files_to_test_doc
+
+__version__ = "0.0.1"
+__name__ = "omni_split"
+__author__ = "dinobot22"
+
+__all__ = [
+    "__name__",
+    "__version__",
+    "__author__",
+    "OmniSplit",
+    "word_preprocessing_and_return_bytesIO",
+    "download_files_to_test_doc"
+]

omni_split-0.0.1rc0/omni_split/base/chonkie_base.py

@@ -0,0 +1,139 @@
+"""Base classes for chunking text."""
+
+import warnings
+from abc import ABC, abstractmethod
+from multiprocessing import Pool, cpu_count
+from typing import Any, Callable, List, Union
+
+from tqdm import tqdm
+
+from .chonkie_tokenizer import Tokenizer
+from .chonkie_types import Chunk
+
+
+class BaseChunker(ABC):
+    """Abstract base class for all chunker implementations.
+
+    All chunker implementations should inherit from this class and implement
+    the chunk() method according to their specific chunking strategy.
+    """
+
+    def __init__(self, tokenizer_or_token_counter: Union[str, Any, Callable[[str], int]]):
+        """Initialize the chunker with a tokenizer.
+
+        Args:
+            tokenizer_or_token_counter (Union[str, Any]): String, tokenizer object, or token counter object
+
+        """
+        self.tokenizer = Tokenizer(tokenizer_or_token_counter)
+
+        # Set whether to use multiprocessing or not
+        self._use_multiprocessing = True
+
+    @abstractmethod
+    def chunk(self, text: str) -> List[Chunk]:
+        """Split text into chunks according to the implementation strategy.
+
+        Args:
+            text: Input text to be chunked
+
+        Returns:
+            List of Chunk objects containing the chunked text and metadata
+
+        """
+        pass
+
+    def _determine_optimal_workers(self) -> int:
+        """Determine optimal number of workers based on system resources."""
+        try:
+            # Get CPU cores
+            cpu_cores = cpu_count()
+
+            # Never use more than 75% of available cores
+            max_workers = max(1, int(cpu_cores * 0.75))
+
+            # Cap at 8 workers
+            return min(max_workers, 8)
+
+        except Exception as e:
+            warnings.warn(f"Error determining optimal workers: {e}. Using single process.")
+            return 1
+
+    def _process_batch_sequential(self, texts: List[str], show_progress_bar: bool = True) -> List[List[Chunk]]:
+        """Process a batch of texts sequentially."""
+        return [
+            self.chunk(t)
+            for t in tqdm(
+                texts,
+                desc="🦛",
+                disable=not show_progress_bar,
+                unit="doc",
+                bar_format="{desc} ch{bar:20}nk {percentage:3.0f}% • {n_fmt}/{total_fmt} docs chunked [{elapsed}<{remaining}, {rate_fmt}] 🌱",
+                ascii=" o",
+            )
+        ]
+
+    def _process_batch_multiprocessing(self, texts: List[str], show_progress_bar: bool = True) -> List[List[Chunk]]:
+        """Process a batch of texts using multiprocessing."""
+        num_workers = self._determine_optimal_workers()
+        total = len(texts)
+        chunksize = max(1, min(total // (num_workers * 16), 10))  # Optimize chunk size
+
+        with Pool(processes=num_workers) as pool:
+            results = []
+            with tqdm(
+                total=total,
+                desc="🦛",
+                disable=not show_progress_bar,
+                unit="doc",
+                bar_format="{desc} ch{bar:20}nk {percentage:3.0f}% • {n_fmt}/{total_fmt} docs chunked [{elapsed}<{remaining}, {rate_fmt}] 🌱",
+                ascii=" o",
+            ) as pbar:
+                for result in pool.imap(self.chunk, texts, chunksize=chunksize):
+                    results.append(result)
+                    pbar.update()
+            return results
+
+    def chunk_batch(
+        self,
+        texts: List[str],
+        show_progress_bar: bool = True,
+    ) -> List[List[Chunk]]:
+        """Split a List of texts into their respective chunks.
+
+        By default, this method uses multiprocessing to parallelize the chunking process.
+
+        Args:
+            texts: List of input texts to be chunked.
+            show_progress_bar: Whether to show a progress bar.
+
+        Returns:
+            List of lists of Chunk objects containing the chunked text and metadata
+
+        """
+        if self._use_multiprocessing:
+            return self._process_batch_multiprocessing(texts, show_progress_bar)
+        else:
+            return self._process_batch_sequential(texts, show_progress_bar)
+
+    def __call__(self, text: Union[str, List[str]], show_progress_bar: bool = True) -> Union[List[Chunk], List[List[Chunk]]]:
+        """Make the chunker callable directly.
+
+        Args:
+            text: Input text or list of texts to be chunked
+            show_progress_bar: Whether to show a progress bar (for batch chunking)
+
+        Returns:
+            List of Chunk objects or list of lists of Chunk
+
+        """
+        if isinstance(text, str):
+            return self.chunk(text)
+        elif isinstance(text, list):
+            return self.chunk_batch(text, show_progress_bar)
+        else:
+            raise ValueError("Input must be a string or a list of strings.")
+
+    def __repr__(self) -> str:
+        """Return string representation of the chunker."""
+        return f"{self.__class__.__name__}()"