hamtaa-texttools 1.1.22__tar.gz → 1.2.0__tar.gz

{hamtaa_texttools-1.1.22/hamtaa_texttools.egg-info → hamtaa_texttools-1.2.0}/PKG-INFO

@@ -1,34 +1,20 @@
 Metadata-Version: 2.4
 Name: hamtaa-texttools
-Version: 1.1.22
+Version: 1.2.0
 Summary: A high-level NLP toolkit built on top of modern LLMs.
-Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, MoosaviNejad <erfanmoosavi84@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
-License: MIT License
-        
-        Copyright (c) 2025 Hamtaa
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
+Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Erfan Moosavi <erfanmoosavi84@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
+Maintainer-email: Erfan Moosavi <erfanmoosavi84@gmail.com>, Tohidi <the.mohammad.tohidi@gmail.com>
+License: MIT
 Keywords: nlp,llm,text-processing,openai
-Requires-Python: >=3.8
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: openai==1.97.1
+Requires-Dist: openai>=1.97.1
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pyyaml>=6.0
 Dynamic: license-file

{hamtaa_texttools-1.1.22 → hamtaa_texttools-1.2.0/hamtaa_texttools.egg-info}/PKG-INFO

@@ -1,34 +1,20 @@
 Metadata-Version: 2.4
 Name: hamtaa-texttools
-Version: 1.1.22
+Version: 1.2.0
 Summary: A high-level NLP toolkit built on top of modern LLMs.
-Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, MoosaviNejad <erfanmoosavi84@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
-License: MIT License
-        
-        Copyright (c) 2025 Hamtaa
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
+Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Erfan Moosavi <erfanmoosavi84@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
+Maintainer-email: Erfan Moosavi <erfanmoosavi84@gmail.com>, Tohidi <the.mohammad.tohidi@gmail.com>
+License: MIT
 Keywords: nlp,llm,text-processing,openai
-Requires-Python: >=3.8
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: openai==1.97.1
+Requires-Dist: openai>=1.97.1
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pyyaml>=6.0
 Dynamic: license-file

{hamtaa_texttools-1.1.22 → hamtaa_texttools-1.2.0}/hamtaa_texttools.egg-info/SOURCES.txt

@@ -1,5 +1,4 @@
 LICENSE
-MANIFEST.in
 README.md
 pyproject.toml
 hamtaa_texttools.egg-info/PKG-INFO
@@ -11,17 +10,18 @@ tests/test_all_async_tools.py
 tests/test_all_tools.py
 tests/test_output_validation.py
 texttools/__init__.py
-texttools/batch/batch_config.py
-texttools/batch/batch_manager.py
-texttools/batch/batch_runner.py
-texttools/internals/async_operator.py
-texttools/internals/exceptions.py
-texttools/internals/models.py
-texttools/internals/operator_utils.py
-texttools/internals/prompt_loader.py
-texttools/internals/sync_operator.py
-texttools/internals/text_to_chunks.py
-texttools/prompts/README.md
+texttools/models.py
+texttools/py.typed
+texttools/batch/__init__.py
+texttools/batch/config.py
+texttools/batch/manager.py
+texttools/batch/runner.py
+texttools/core/__init__.py
+texttools/core/engine.py
+texttools/core/exceptions.py
+texttools/core/internal_models.py
+texttools/core/operators/async_operator.py
+texttools/core/operators/sync_operator.py
 texttools/prompts/categorize.yaml
 texttools/prompts/check_fact.yaml
 texttools/prompts/extract_entities.yaml
@@ -35,5 +35,6 @@ texttools/prompts/subject_to_question.yaml
 texttools/prompts/summarize.yaml
 texttools/prompts/text_to_question.yaml
 texttools/prompts/translate.yaml
+texttools/tools/__init__.py
 texttools/tools/async_tools.py
 texttools/tools/sync_tools.py

{hamtaa_texttools-1.1.22 → hamtaa_texttools-1.2.0}/hamtaa_texttools.egg-info/requires.txt

@@ -1,3 +1,3 @@
-openai==1.97.1
+openai>=1.97.1
 pydantic>=2.0.0
 pyyaml>=6.0

hamtaa_texttools-1.2.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hamtaa-texttools"
+version = "1.2.0"
+authors = [
+  {name = "Tohidi", email = "the.mohammad.tohidi@gmail.com"},
+  {name = "Erfan Moosavi", email = "erfanmoosavi84@gmail.com"},
+  {name = "Montazer", email = "montazerh82@gmail.com"},
+  {name = "Givechi", email = "mohamad.m.givechi@gmail.com"},
+  {name = "Zareshahi", email = "a.zareshahi1377@gmail.com"},
+]
+maintainers = [
+    {name = "Erfan Moosavi", email = "erfanmoosavi84@gmail.com"},
+    {name = "Tohidi", email = "the.mohammad.tohidi@gmail.com"},
+]
+description = "A high-level NLP toolkit built on top of modern LLMs."
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+dependencies = [
+  "openai>=1.97.1",
+  "pydantic>=2.0.0",
+  "pyyaml>=6.0",
+]
+keywords = ["nlp", "llm", "text-processing", "openai"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Text Processing",
+    "Operating System :: OS Independent",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["texttools*"]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+"texttools" = ["prompts/*.yaml", "py.typed"]

{hamtaa_texttools-1.1.22 → hamtaa_texttools-1.2.0}/tests/test_all_async_tools.py

@@ -6,16 +6,13 @@ from openai import AsyncOpenAI
 
 from texttools import AsyncTheTool
 
-# Load environment variables from .env
 load_dotenv()
-API_KEY = os.getenv("OPENAI_API_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 BASE_URL = os.getenv("BASE_URL")
 MODEL = os.getenv("MODEL")
 
-# Create AsyncOpenAI client
-client = AsyncOpenAI(base_url=BASE_URL, api_key=API_KEY)
+client = AsyncOpenAI(base_url=BASE_URL, api_key=OPENAI_API_KEY)
 
-# Create an instance of TheTool
 t = AsyncTheTool(client=client, model=MODEL)
 
 

{hamtaa_texttools-1.1.22 → hamtaa_texttools-1.2.0}/tests/test_all_tools.py

@@ -6,24 +6,18 @@ from pydantic import BaseModel
 
 from texttools import TheTool, CategoryTree
 
-# Load environment variables from .env
 load_dotenv()
-API_KEY = os.getenv("OPENAI_API_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 BASE_URL = os.getenv("BASE_URL")
 MODEL = os.getenv("MODEL")
 
-# Create OpenAI client
-client = OpenAI(base_url=BASE_URL, api_key=API_KEY)
+client = OpenAI(base_url=BASE_URL, api_key=OPENAI_API_KEY)
 
-# Create an instance of TheTool
 t = TheTool(client=client, model=MODEL)
 
 # Categorizer: list mode
 category = t.categorize(
-    "سلام حالت چطوره؟",
-    categories=["هیچکدام", "دینی", "فلسفه"],
-    logprobs=True,
-    top_logprobs=3,
+    "سلام حالت چطوره؟", categories=["هیچکدام", "دینی", "فلسفه"], priority=3
 )
 print(repr(category))
 
@@ -56,7 +50,6 @@ entities = t.extract_entities(
     "Ali will be dead by the car crash",
     entities=["EVENT"],
     with_analysis=True,
-    logprobs=True,
 )
 print(repr(entities))
 

{hamtaa_texttools-1.1.22 → hamtaa_texttools-1.2.0}/tests/test_output_validation.py

@@ -6,16 +6,13 @@ from openai import OpenAI
 
 from texttools import TheTool
 
-# Load environment variables from .env
 load_dotenv()
-API_KEY = os.getenv("OPENAI_API_KEY")
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
 BASE_URL = os.getenv("BASE_URL")
 MODEL = os.getenv("MODEL")
 
-# Create OpenAI client
-client = OpenAI(base_url=BASE_URL, api_key=API_KEY)
+client = OpenAI(base_url=BASE_URL, api_key=OPENAI_API_KEY)
 
-# Create an instance of TheTool
 t = TheTool(client=client, model=MODEL)
 
 
@@ -24,7 +21,6 @@ def validate(result: Any) -> bool:
     return "چیست؟" not in result
 
 
-# Question from Text Generator
 question = t.text_to_question(
     "زندگی",
     output_lang="Persian",

{hamtaa_texttools-1.1.22 → hamtaa_texttools-1.2.0}/texttools/__init__.py

@@ -1,7 +1,7 @@
-from .tools.sync_tools import TheTool
+from .batch.config import BatchConfig
+from .batch.runner import BatchRunner
+from .models import CategoryTree
 from .tools.async_tools import AsyncTheTool
-from .internals.models import CategoryTree
-from .batch.batch_runner import BatchRunner
-from .batch.batch_config import BatchConfig
+from .tools.sync_tools import TheTool
 
 __all__ = ["TheTool", "AsyncTheTool", "CategoryTree", "BatchRunner", "BatchConfig"]

hamtaa_texttools-1.1.22/texttools/batch/batch_config.py → hamtaa_texttools-1.2.0/texttools/batch/config.py

@@ -1,5 +1,6 @@
-from dataclasses import dataclass
 from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
 
 
 def export_data(data) -> list[dict[str, str]]:
@@ -10,7 +11,7 @@ def export_data(data) -> list[dict[str, str]]:
     return data
 
 
-def import_data(data) -> object:
+def import_data(data) -> Any:
     """
     Takes the output and adds and aggregates it to the original structure.
     """

hamtaa_texttools-1.1.22/texttools/batch/batch_manager.py → hamtaa_texttools-1.2.0/texttools/batch/manager.py

@@ -1,12 +1,12 @@
 import json
+import logging
 import uuid
 from pathlib import Path
-from typing import Type, TypeVar
-import logging
+from typing import Any, Type, TypeVar
 
-from pydantic import BaseModel
 from openai import OpenAI
 from openai.lib._pydantic import to_strict_json_schema
+from pydantic import BaseModel
 
 # Base Model type for output models
 T = TypeVar("T", bound=BaseModel)
@@ -31,7 +31,7 @@ class BatchManager:
         prompt_template: str,
         state_dir: Path = Path(".batch_jobs"),
         custom_json_schema_obj_str: dict | None = None,
-        **client_kwargs: object,
+        **client_kwargs: Any,
     ):
         self._client = client
         self._model = model
@@ -51,7 +51,7 @@ class BatchManager:
     def _state_file(self, job_name: str) -> Path:
         return self._state_dir / f"{job_name}.json"
 
-    def _load_state(self, job_name: str) -> list[dict[str, object]]:
+    def _load_state(self, job_name: str) -> list[dict[str, Any]]:
         """
         Loads the state (job information) from the state file for the given job name.
         Returns an empty list if the state file does not exist.
@@ -62,7 +62,7 @@ class BatchManager:
                 return json.load(f)
         return []
 
-    def _save_state(self, job_name: str, jobs: list[dict[str, object]]) -> None:
+    def _save_state(self, job_name: str, jobs: list[dict[str, Any]]) -> None:
         """
         Saves the job state to the state file for the given job name.
         """
@@ -77,11 +77,11 @@ class BatchManager:
         if path.exists():
             path.unlink()
 
-    def _build_task(self, text: str, idx: str) -> dict[str, object]:
+    def _build_task(self, text: str, idx: str) -> dict[str, Any]:
         """
         Builds a single task dictionary for the batch job, including the prompt, model, and response format configuration.
         """
-        response_format_config: dict[str, object]
+        response_format_config: dict[str, Any]
 
         if self._custom_json_schema_obj_str:
             response_format_config = {

hamtaa_texttools-1.1.22/texttools/batch/batch_runner.py → hamtaa_texttools-1.2.0/texttools/batch/runner.py

@@ -1,18 +1,18 @@
 import json
+import logging
 import os
 import time
 from pathlib import Path
-from typing import Type, TypeVar
-import logging
+from typing import Any, Type, TypeVar
 
 from dotenv import load_dotenv
 from openai import OpenAI
 from pydantic import BaseModel
 
-from texttools.batch.batch_manager import BatchManager
-from texttools.batch.batch_config import BatchConfig
-from texttools.internals.models import Str
-from texttools.internals.exceptions import TextToolsError, ConfigurationError
+from ..core.exceptions import TextToolsError
+from ..core.internal_models import Str
+from .config import BatchConfig
+from .manager import BatchManager
 
 # Base Model type for output models
 T = TypeVar("T", bound=BaseModel)
@@ -38,7 +38,7 @@ class BatchRunner:
             self._output_model = output_model
             self._manager = self._init_manager()
             self._data = self._load_data()
-            self._parts: list[list[dict[str, object]]] = []
+            self._parts: list[list[dict[str, Any]]] = []
             # Map part index to job name
             self._part_idx_to_job_name: dict[int, str] = {}
             # Track retry attempts per part
@@ -47,7 +47,7 @@ class BatchRunner:
             Path(self._config.BASE_OUTPUT_DIR).mkdir(parents=True, exist_ok=True)
 
         except Exception as e:
-            raise ConfigurationError(f"Batch runner initialization failed: {e}")
+            raise TextToolsError(f"Batch runner initialization failed: {e}")
 
     def _init_manager(self) -> BatchManager:
         load_dotenv()
@@ -130,8 +130,8 @@ class BatchRunner:
 
     def _save_results(
         self,
-        output_data: list[dict[str, object]] | dict[str, object],
-        log: list[object],
+        output_data: list[dict[str, Any]] | dict[str, Any],
+        log: list[Any],
         part_idx: int,
     ):
         part_suffix = f"_part_{part_idx + 1}" if len(self._parts) > 1 else ""

hamtaa_texttools-1.2.0/texttools/core/engine.py

@@ -0,0 +1,254 @@
+import math
+import random
+import re
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+from .exceptions import PromptError
+
+
+class PromptLoader:
+    """
+    Utility for loading and formatting YAML prompt templates.
+    """
+
+    MAIN_TEMPLATE = "main_template"
+    ANALYZE_TEMPLATE = "analyze_template"
+
+    @lru_cache(maxsize=32)
+    def _load_templates(self, prompt_file: str, mode: str | None) -> dict[str, str]:
+        """
+        Loads prompt templates from YAML file with optional mode selection.
+        """
+        try:
+            base_dir = Path(__file__).parent.parent / Path("prompts")
+            prompt_path = base_dir / prompt_file
+
+            if not prompt_path.exists():
+                raise PromptError(f"Prompt file not found: {prompt_file}")
+
+            data = yaml.safe_load(prompt_path.read_text(encoding="utf-8"))
+
+            if self.MAIN_TEMPLATE not in data:
+                raise PromptError(f"Missing 'main_template' in {prompt_file}")
+
+            if self.ANALYZE_TEMPLATE not in data:
+                raise PromptError(f"Missing 'analyze_template' in {prompt_file}")
+
+            if mode and mode not in data.get(self.MAIN_TEMPLATE, {}):
+                raise PromptError(f"Mode '{mode}' not found in {prompt_file}")
+
+            main_template = (
+                data[self.MAIN_TEMPLATE][mode]
+                if mode and isinstance(data[self.MAIN_TEMPLATE], dict)
+                else data[self.MAIN_TEMPLATE]
+            )
+
+            analyze_template = (
+                data[self.ANALYZE_TEMPLATE][mode]
+                if mode and isinstance(data[self.ANALYZE_TEMPLATE], dict)
+                else data[self.ANALYZE_TEMPLATE]
+            )
+
+            if not main_template or not main_template.strip():
+                raise PromptError(
+                    f"Empty main_template in {prompt_file}"
+                    + (f" for mode '{mode}'" if mode else "")
+                )
+
+            return {
+                self.MAIN_TEMPLATE: main_template,
+                self.ANALYZE_TEMPLATE: analyze_template,
+            }
+
+        except yaml.YAMLError as e:
+            raise PromptError(f"Invalid YAML in {prompt_file}: {e}")
+        except Exception as e:
+            raise PromptError(f"Failed to load prompt {prompt_file}: {e}")
+
+    def load(
+        self, prompt_file: str, text: str, mode: str, **extra_kwargs
+    ) -> dict[str, str]:
+        try:
+            template_configs = self._load_templates(prompt_file, mode)
+            format_args = {"text": text}
+            format_args.update(extra_kwargs)
+
+            # Inject variables inside each template
+            for key in template_configs.keys():
+                template_configs[key] = template_configs[key].format(**format_args)
+
+            return template_configs
+
+        except KeyError as e:
+            raise PromptError(f"Missing template variable: {e}")
+        except Exception as e:
+            raise PromptError(f"Failed to format prompt: {e}")
+
+
+class OperatorUtils:
+    @staticmethod
+    def build_main_prompt(
+        main_template: str,
+        analysis: str | None,
+        output_lang: str | None,
+        user_prompt: str | None,
+    ) -> str:
+        main_prompt = ""
+
+        if analysis:
+            main_prompt += f"Based on this analysis:\n{analysis}\n"
+
+        if output_lang:
+            main_prompt += f"Respond only in the {output_lang} language.\n"
+
+        if user_prompt:
+            main_prompt += f"Consider this instruction {user_prompt}\n"
+
+        main_prompt += main_template
+
+        return main_prompt
+
+    @staticmethod
+    def build_message(prompt: str) -> list[dict[str, str]]:
+        return [{"role": "user", "content": prompt}]
+
+    @staticmethod
+    def extract_logprobs(completion: dict) -> list[dict]:
+        """
+        Extracts and filters token probabilities from completion logprobs.
+        Skips punctuation and structural tokens, returns cleaned probability data.
+        """
+        logprobs_data = []
+
+        ignore_pattern = re.compile(r'^(result|[\s\[\]\{\}",:]+)$')
+
+        for choice in completion.choices:
+            if not getattr(choice, "logprobs", None):
+                raise ValueError("Your model does not support logprobs")
+
+            for logprob_item in choice.logprobs.content:
+                if ignore_pattern.match(logprob_item.token):
+                    continue
+                token_entry = {
+                    "token": logprob_item.token,
+                    "prob": round(math.exp(logprob_item.logprob), 8),
+                    "top_alternatives": [],
+                }
+                for alt in logprob_item.top_logprobs:
+                    if ignore_pattern.match(alt.token):
+                        continue
+                    token_entry["top_alternatives"].append(
+                        {
+                            "token": alt.token,
+                            "prob": round(math.exp(alt.logprob), 8),
+                        }
+                    )
+                logprobs_data.append(token_entry)
+
+        return logprobs_data
+
+    @staticmethod
+    def get_retry_temp(base_temp: float) -> float:
+        delta_temp = random.choice([-1, 1]) * random.uniform(0.1, 0.9)
+        new_temp = base_temp + delta_temp
+
+        return max(0.0, min(new_temp, 1.5))
+
+
+def text_to_chunks(text: str, size: int, overlap: int) -> list[str]:
+    separators = ["\n\n", "\n", " ", ""]
+    is_separator_regex = False
+    keep_separator = True  # Equivalent to 'start'
+    length_function = len
+    strip_whitespace = True
+    chunk_size = size
+    chunk_overlap = overlap
+
+    def _split_text_with_regex(
+        text: str, separator: str, keep_separator: bool
+    ) -> list[str]:
+        if not separator:
+            return [text]
+        if not keep_separator:
+            return re.split(separator, text)
+        _splits = re.split(f"({separator})", text)
+        splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)]
+        if len(_splits) % 2 == 0:
+            splits += [_splits[-1]]
+        return [_splits[0]] + splits if _splits[0] else splits
+
+    def _join_docs(docs: list[str], separator: str) -> str | None:
+        text = separator.join(docs)
+        if strip_whitespace:
+            text = text.strip()
+        return text if text else None
+
+    def _merge_splits(splits: list[str], separator: str) -> list[str]:
+        separator_len = length_function(separator)
+        docs = []
+        current_doc = []
+        total = 0
+        for d in splits:
+            len_ = length_function(d)
+            if total + len_ + (separator_len if current_doc else 0) > chunk_size:
+                if total > chunk_size:
+                    pass
+                if current_doc:
+                    doc = _join_docs(current_doc, separator)
+                    if doc is not None:
+                        docs.append(doc)
+                    while total > chunk_overlap or (
+                        total + len_ + (separator_len if current_doc else 0)
+                        > chunk_size
+                        and total > 0
+                    ):
+                        total -= length_function(current_doc[0]) + (
+                            separator_len if len(current_doc) > 1 else 0
+                        )
+                        current_doc = current_doc[1:]
+            current_doc.append(d)
+            total += len_ + (separator_len if len(current_doc) > 1 else 0)
+        doc = _join_docs(current_doc, separator)
+        if doc is not None:
+            docs.append(doc)
+        return docs
+
+    def _split_text(text: str, separators: list[str]) -> list[str]:
+        final_chunks = []
+        separator = separators[-1]
+        new_separators = []
+        for i, _s in enumerate(separators):
+            separator_ = _s if is_separator_regex else re.escape(_s)
+            if not _s:
+                separator = _s
+                break
+            if re.search(separator_, text):
+                separator = _s
+                new_separators = separators[i + 1 :]
+                break
+        separator_ = separator if is_separator_regex else re.escape(separator)
+        splits = _split_text_with_regex(text, separator_, keep_separator)
+        _separator = "" if keep_separator else separator
+        good_splits = []
+        for s in splits:
+            if length_function(s) < chunk_size:
+                good_splits.append(s)
+            else:
+                if good_splits:
+                    merged_text = _merge_splits(good_splits, _separator)
+                    final_chunks.extend(merged_text)
+                    good_splits = []
+                if not new_separators:
+                    final_chunks.append(s)
+                else:
+                    other_info = _split_text(s, new_separators)
+                    final_chunks.extend(other_info)
+        if good_splits:
+            merged_text = _merge_splits(good_splits, _separator)
+            final_chunks.extend(merged_text)
+        return final_chunks
+
+    return _split_text(text, separators)