hamtaa-texttools 1.1.23__tar.gz → 1.2.0__tar.gz

{hamtaa_texttools-1.1.23/hamtaa_texttools.egg-info → hamtaa_texttools-1.2.0}/PKG-INFO

@@ -1,34 +1,20 @@
 Metadata-Version: 2.4
 Name: hamtaa-texttools
-Version: 1.1.23
+Version: 1.2.0
 Summary: A high-level NLP toolkit built on top of modern LLMs.
-Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, MoosaviNejad <erfanmoosavi84@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
-License: MIT License
-        
-        Copyright (c) 2025 Hamtaa
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
+Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Erfan Moosavi <erfanmoosavi84@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
+Maintainer-email: Erfan Moosavi <erfanmoosavi84@gmail.com>, Tohidi <the.mohammad.tohidi@gmail.com>
+License: MIT
 Keywords: nlp,llm,text-processing,openai
-Requires-Python: >=3.8
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: openai==1.97.1
+Requires-Dist: openai>=1.97.1
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pyyaml>=6.0
 Dynamic: license-file

{hamtaa_texttools-1.1.23 → hamtaa_texttools-1.2.0/hamtaa_texttools.egg-info}/PKG-INFO

@@ -1,34 +1,20 @@
 Metadata-Version: 2.4
 Name: hamtaa-texttools
-Version: 1.1.23
+Version: 1.2.0
 Summary: A high-level NLP toolkit built on top of modern LLMs.
-Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, MoosaviNejad <erfanmoosavi84@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
-License: MIT License
-        
-        Copyright (c) 2025 Hamtaa
-        
-        Permission is hereby granted, free of charge, to any person obtaining a copy
-        of this software and associated documentation files (the "Software"), to deal
-        in the Software without restriction, including without limitation the rights
-        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-        copies of the Software, and to permit persons to whom the Software is
-        furnished to do so, subject to the following conditions:
-        
-        The above copyright notice and this permission notice shall be included in all
-        copies or substantial portions of the Software.
-        
-        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-        SOFTWARE.
+Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Erfan Moosavi <erfanmoosavi84@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, Zareshahi <a.zareshahi1377@gmail.com>
+Maintainer-email: Erfan Moosavi <erfanmoosavi84@gmail.com>, Tohidi <the.mohammad.tohidi@gmail.com>
+License: MIT
 Keywords: nlp,llm,text-processing,openai
-Requires-Python: >=3.8
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Text Processing
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.9
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: openai==1.97.1
+Requires-Dist: openai>=1.97.1
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: pyyaml>=6.0
 Dynamic: license-file

{hamtaa_texttools-1.1.23 → hamtaa_texttools-1.2.0}/hamtaa_texttools.egg-info/SOURCES.txt

@@ -1,5 +1,4 @@
 LICENSE
-MANIFEST.in
 README.md
 pyproject.toml
 hamtaa_texttools.egg-info/PKG-INFO
@@ -11,17 +10,18 @@ tests/test_all_async_tools.py
 tests/test_all_tools.py
 tests/test_output_validation.py
 texttools/__init__.py
-texttools/batch/batch_config.py
-texttools/batch/batch_manager.py
-texttools/batch/batch_runner.py
-texttools/internals/async_operator.py
-texttools/internals/exceptions.py
-texttools/internals/models.py
-texttools/internals/operator_utils.py
-texttools/internals/prompt_loader.py
-texttools/internals/sync_operator.py
-texttools/internals/text_to_chunks.py
-texttools/prompts/README.md
+texttools/models.py
+texttools/py.typed
+texttools/batch/__init__.py
+texttools/batch/config.py
+texttools/batch/manager.py
+texttools/batch/runner.py
+texttools/core/__init__.py
+texttools/core/engine.py
+texttools/core/exceptions.py
+texttools/core/internal_models.py
+texttools/core/operators/async_operator.py
+texttools/core/operators/sync_operator.py
 texttools/prompts/categorize.yaml
 texttools/prompts/check_fact.yaml
 texttools/prompts/extract_entities.yaml
@@ -35,5 +35,6 @@ texttools/prompts/subject_to_question.yaml
 texttools/prompts/summarize.yaml
 texttools/prompts/text_to_question.yaml
 texttools/prompts/translate.yaml
+texttools/tools/__init__.py
 texttools/tools/async_tools.py
 texttools/tools/sync_tools.py

{hamtaa_texttools-1.1.23 → hamtaa_texttools-1.2.0}/hamtaa_texttools.egg-info/requires.txt

@@ -1,3 +1,3 @@
-openai==1.97.1
+openai>=1.97.1
 pydantic>=2.0.0
 pyyaml>=6.0

hamtaa_texttools-1.2.0/pyproject.toml

@@ -0,0 +1,45 @@
+[build-system]
+requires = ["setuptools>=61.0", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "hamtaa-texttools"
+version = "1.2.0"
+authors = [
+  {name = "Tohidi", email = "the.mohammad.tohidi@gmail.com"},
+  {name = "Erfan Moosavi", email = "erfanmoosavi84@gmail.com"},
+  {name = "Montazer", email = "montazerh82@gmail.com"},
+  {name = "Givechi", email = "mohamad.m.givechi@gmail.com"},
+  {name = "Zareshahi", email = "a.zareshahi1377@gmail.com"},
+]
+maintainers = [
+    {name = "Erfan Moosavi", email = "erfanmoosavi84@gmail.com"},
+    {name = "Tohidi", email = "the.mohammad.tohidi@gmail.com"},
+]
+description = "A high-level NLP toolkit built on top of modern LLMs."
+readme = "README.md"
+license = {text = "MIT"}
+requires-python = ">=3.9"
+dependencies = [
+  "openai>=1.97.1",
+  "pydantic>=2.0.0",
+  "pyyaml>=6.0",
+]
+keywords = ["nlp", "llm", "text-processing", "openai"]
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: MIT License",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "Topic :: Text Processing",
+    "Operating System :: OS Independent",
+]
+
+[tool.setuptools.packages.find]
+where = ["."]
+include = ["texttools*"]
+
+[tool.setuptools]
+include-package-data = true
+
+[tool.setuptools.package-data]
+"texttools" = ["prompts/*.yaml", "py.typed"]

{hamtaa_texttools-1.1.23 → hamtaa_texttools-1.2.0}/texttools/__init__.py

@@ -1,7 +1,7 @@
-from .tools.sync_tools import TheTool
+from .batch.config import BatchConfig
+from .batch.runner import BatchRunner
+from .models import CategoryTree
 from .tools.async_tools import AsyncTheTool
-from .internals.models import CategoryTree
-from .batch.batch_runner import BatchRunner
-from .batch.batch_config import BatchConfig
+from .tools.sync_tools import TheTool
 
 __all__ = ["TheTool", "AsyncTheTool", "CategoryTree", "BatchRunner", "BatchConfig"]

hamtaa_texttools-1.1.23/texttools/batch/batch_config.py → hamtaa_texttools-1.2.0/texttools/batch/config.py

@@ -1,6 +1,6 @@
-from typing import Any
-from dataclasses import dataclass
 from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any
 
 
 def export_data(data) -> list[dict[str, str]]:

hamtaa_texttools-1.1.23/texttools/batch/batch_manager.py → hamtaa_texttools-1.2.0/texttools/batch/manager.py

@@ -1,12 +1,12 @@
 import json
+import logging
 import uuid
 from pathlib import Path
-from typing import Type, TypeVar, Any
-import logging
+from typing import Any, Type, TypeVar
 
-from pydantic import BaseModel
 from openai import OpenAI
 from openai.lib._pydantic import to_strict_json_schema
+from pydantic import BaseModel
 
 # Base Model type for output models
 T = TypeVar("T", bound=BaseModel)

hamtaa_texttools-1.1.23/texttools/batch/batch_runner.py → hamtaa_texttools-1.2.0/texttools/batch/runner.py

@@ -1,18 +1,18 @@
 import json
+import logging
 import os
 import time
 from pathlib import Path
-from typing import Type, TypeVar, Any
-import logging
+from typing import Any, Type, TypeVar
 
 from dotenv import load_dotenv
 from openai import OpenAI
 from pydantic import BaseModel
 
-from texttools.batch.batch_manager import BatchManager
-from texttools.batch.batch_config import BatchConfig
-from texttools.internals.models import Str
-from texttools.internals.exceptions import TextToolsError
+from ..core.exceptions import TextToolsError
+from ..core.internal_models import Str
+from .config import BatchConfig
+from .manager import BatchManager
 
 # Base Model type for output models
 T = TypeVar("T", bound=BaseModel)

hamtaa_texttools-1.2.0/texttools/core/engine.py

@@ -0,0 +1,254 @@
+import math
+import random
+import re
+from functools import lru_cache
+from pathlib import Path
+
+import yaml
+
+from .exceptions import PromptError
+
+
+class PromptLoader:
+    """
+    Utility for loading and formatting YAML prompt templates.
+    """
+
+    MAIN_TEMPLATE = "main_template"
+    ANALYZE_TEMPLATE = "analyze_template"
+
+    @lru_cache(maxsize=32)
+    def _load_templates(self, prompt_file: str, mode: str | None) -> dict[str, str]:
+        """
+        Loads prompt templates from YAML file with optional mode selection.
+        """
+        try:
+            base_dir = Path(__file__).parent.parent / Path("prompts")
+            prompt_path = base_dir / prompt_file
+
+            if not prompt_path.exists():
+                raise PromptError(f"Prompt file not found: {prompt_file}")
+
+            data = yaml.safe_load(prompt_path.read_text(encoding="utf-8"))
+
+            if self.MAIN_TEMPLATE not in data:
+                raise PromptError(f"Missing 'main_template' in {prompt_file}")
+
+            if self.ANALYZE_TEMPLATE not in data:
+                raise PromptError(f"Missing 'analyze_template' in {prompt_file}")
+
+            if mode and mode not in data.get(self.MAIN_TEMPLATE, {}):
+                raise PromptError(f"Mode '{mode}' not found in {prompt_file}")
+
+            main_template = (
+                data[self.MAIN_TEMPLATE][mode]
+                if mode and isinstance(data[self.MAIN_TEMPLATE], dict)
+                else data[self.MAIN_TEMPLATE]
+            )
+
+            analyze_template = (
+                data[self.ANALYZE_TEMPLATE][mode]
+                if mode and isinstance(data[self.ANALYZE_TEMPLATE], dict)
+                else data[self.ANALYZE_TEMPLATE]
+            )
+
+            if not main_template or not main_template.strip():
+                raise PromptError(
+                    f"Empty main_template in {prompt_file}"
+                    + (f" for mode '{mode}'" if mode else "")
+                )
+
+            return {
+                self.MAIN_TEMPLATE: main_template,
+                self.ANALYZE_TEMPLATE: analyze_template,
+            }
+
+        except yaml.YAMLError as e:
+            raise PromptError(f"Invalid YAML in {prompt_file}: {e}")
+        except Exception as e:
+            raise PromptError(f"Failed to load prompt {prompt_file}: {e}")
+
+    def load(
+        self, prompt_file: str, text: str, mode: str, **extra_kwargs
+    ) -> dict[str, str]:
+        try:
+            template_configs = self._load_templates(prompt_file, mode)
+            format_args = {"text": text}
+            format_args.update(extra_kwargs)
+
+            # Inject variables inside each template
+            for key in template_configs.keys():
+                template_configs[key] = template_configs[key].format(**format_args)
+
+            return template_configs
+
+        except KeyError as e:
+            raise PromptError(f"Missing template variable: {e}")
+        except Exception as e:
+            raise PromptError(f"Failed to format prompt: {e}")
+
+
+class OperatorUtils:
+    @staticmethod
+    def build_main_prompt(
+        main_template: str,
+        analysis: str | None,
+        output_lang: str | None,
+        user_prompt: str | None,
+    ) -> str:
+        main_prompt = ""
+
+        if analysis:
+            main_prompt += f"Based on this analysis:\n{analysis}\n"
+
+        if output_lang:
+            main_prompt += f"Respond only in the {output_lang} language.\n"
+
+        if user_prompt:
+            main_prompt += f"Consider this instruction {user_prompt}\n"
+
+        main_prompt += main_template
+
+        return main_prompt
+
+    @staticmethod
+    def build_message(prompt: str) -> list[dict[str, str]]:
+        return [{"role": "user", "content": prompt}]
+
+    @staticmethod
+    def extract_logprobs(completion: dict) -> list[dict]:
+        """
+        Extracts and filters token probabilities from completion logprobs.
+        Skips punctuation and structural tokens, returns cleaned probability data.
+        """
+        logprobs_data = []
+
+        ignore_pattern = re.compile(r'^(result|[\s\[\]\{\}",:]+)$')
+
+        for choice in completion.choices:
+            if not getattr(choice, "logprobs", None):
+                raise ValueError("Your model does not support logprobs")
+
+            for logprob_item in choice.logprobs.content:
+                if ignore_pattern.match(logprob_item.token):
+                    continue
+                token_entry = {
+                    "token": logprob_item.token,
+                    "prob": round(math.exp(logprob_item.logprob), 8),
+                    "top_alternatives": [],
+                }
+                for alt in logprob_item.top_logprobs:
+                    if ignore_pattern.match(alt.token):
+                        continue
+                    token_entry["top_alternatives"].append(
+                        {
+                            "token": alt.token,
+                            "prob": round(math.exp(alt.logprob), 8),
+                        }
+                    )
+                logprobs_data.append(token_entry)
+
+        return logprobs_data
+
+    @staticmethod
+    def get_retry_temp(base_temp: float) -> float:
+        delta_temp = random.choice([-1, 1]) * random.uniform(0.1, 0.9)
+        new_temp = base_temp + delta_temp
+
+        return max(0.0, min(new_temp, 1.5))
+
+
+def text_to_chunks(text: str, size: int, overlap: int) -> list[str]:
+    separators = ["\n\n", "\n", " ", ""]
+    is_separator_regex = False
+    keep_separator = True  # Equivalent to 'start'
+    length_function = len
+    strip_whitespace = True
+    chunk_size = size
+    chunk_overlap = overlap
+
+    def _split_text_with_regex(
+        text: str, separator: str, keep_separator: bool
+    ) -> list[str]:
+        if not separator:
+            return [text]
+        if not keep_separator:
+            return re.split(separator, text)
+        _splits = re.split(f"({separator})", text)
+        splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)]
+        if len(_splits) % 2 == 0:
+            splits += [_splits[-1]]
+        return [_splits[0]] + splits if _splits[0] else splits
+
+    def _join_docs(docs: list[str], separator: str) -> str | None:
+        text = separator.join(docs)
+        if strip_whitespace:
+            text = text.strip()
+        return text if text else None
+
+    def _merge_splits(splits: list[str], separator: str) -> list[str]:
+        separator_len = length_function(separator)
+        docs = []
+        current_doc = []
+        total = 0
+        for d in splits:
+            len_ = length_function(d)
+            if total + len_ + (separator_len if current_doc else 0) > chunk_size:
+                if total > chunk_size:
+                    pass
+                if current_doc:
+                    doc = _join_docs(current_doc, separator)
+                    if doc is not None:
+                        docs.append(doc)
+                    while total > chunk_overlap or (
+                        total + len_ + (separator_len if current_doc else 0)
+                        > chunk_size
+                        and total > 0
+                    ):
+                        total -= length_function(current_doc[0]) + (
+                            separator_len if len(current_doc) > 1 else 0
+                        )
+                        current_doc = current_doc[1:]
+            current_doc.append(d)
+            total += len_ + (separator_len if len(current_doc) > 1 else 0)
+        doc = _join_docs(current_doc, separator)
+        if doc is not None:
+            docs.append(doc)
+        return docs
+
+    def _split_text(text: str, separators: list[str]) -> list[str]:
+        final_chunks = []
+        separator = separators[-1]
+        new_separators = []
+        for i, _s in enumerate(separators):
+            separator_ = _s if is_separator_regex else re.escape(_s)
+            if not _s:
+                separator = _s
+                break
+            if re.search(separator_, text):
+                separator = _s
+                new_separators = separators[i + 1 :]
+                break
+        separator_ = separator if is_separator_regex else re.escape(separator)
+        splits = _split_text_with_regex(text, separator_, keep_separator)
+        _separator = "" if keep_separator else separator
+        good_splits = []
+        for s in splits:
+            if length_function(s) < chunk_size:
+                good_splits.append(s)
+            else:
+                if good_splits:
+                    merged_text = _merge_splits(good_splits, _separator)
+                    final_chunks.extend(merged_text)
+                    good_splits = []
+                if not new_separators:
+                    final_chunks.append(s)
+                else:
+                    other_info = _split_text(s, new_separators)
+                    final_chunks.extend(other_info)
+        if good_splits:
+            merged_text = _merge_splits(good_splits, _separator)
+            final_chunks.extend(merged_text)
+        return final_chunks
+
+    return _split_text(text, separators)

hamtaa_texttools-1.2.0/texttools/core/internal_models.py

@@ -0,0 +1,58 @@
+from typing import Any, Literal, Type
+
+from pydantic import BaseModel, Field, create_model
+
+
+class OperatorOutput(BaseModel):
+    result: Any
+    analysis: str | None
+    logprobs: list[dict[str, Any]] | None
+
+
+class Str(BaseModel):
+    result: str = Field(..., description="The output string", example="text")
+
+
+class Bool(BaseModel):
+    result: bool = Field(
+        ..., description="Boolean indicating the output state", example=True
+    )
+
+
+class ListStr(BaseModel):
+    result: list[str] = Field(
+        ..., description="The output list of strings", example=["text_1", "text_2"]
+    )
+
+
+class ListDictStrStr(BaseModel):
+    result: list[dict[str, str]] = Field(
+        ...,
+        description="List of dictionaries containing string key-value pairs",
+        example=[{"text": "Mohammad", "type": "PER"}, {"text": "Iran", "type": "LOC"}],
+    )
+
+
+class ReasonListStr(BaseModel):
+    reason: str = Field(..., description="Thinking process that led to the output")
+    result: list[str] = Field(
+        ..., description="The output list of strings", example=["text_1", "text_2"]
+    )
+
+
+# This function is needed to create CategorizerOutput with dynamic categories
+def create_dynamic_model(allowed_values: list[str]) -> Type[BaseModel]:
+    literal_type = Literal[*allowed_values]
+
+    CategorizerOutput = create_model(
+        "CategorizerOutput",
+        reason=(
+            str,
+            Field(
+                ..., description="Explanation of why the input belongs to the category"
+            ),
+        ),
+        result=(literal_type, Field(..., description="Predicted category label")),
+    )
+
+    return CategorizerOutput

{hamtaa_texttools-1.1.23/texttools/internals → hamtaa_texttools-1.2.0/texttools/core/operators}/async_operator.py

@@ -1,18 +1,12 @@
-from typing import TypeVar, Type, Any
 from collections.abc import Callable
+from typing import Any, Type, TypeVar
 
 from openai import AsyncOpenAI
 from pydantic import BaseModel
 
-from texttools.internals.models import OperatorOutput
-from texttools.internals.operator_utils import OperatorUtils
-from texttools.internals.prompt_loader import PromptLoader
-from texttools.internals.exceptions import (
-    TextToolsError,
-    LLMError,
-    ValidationError,
-    PromptError,
-)
+from ..engine import OperatorUtils, PromptLoader
+from ..exceptions import LLMError, PromptError, TextToolsError, ValidationError
+from ..internal_models import OperatorOutput
 
 # Base Model type for output models
 T = TypeVar("T", bound=BaseModel)

{hamtaa_texttools-1.1.23/texttools/internals → hamtaa_texttools-1.2.0/texttools/core/operators}/sync_operator.py

@@ -1,18 +1,12 @@
-from typing import TypeVar, Type, Any
 from collections.abc import Callable
+from typing import Any, Type, TypeVar
 
 from openai import OpenAI
 from pydantic import BaseModel
 
-from texttools.internals.models import OperatorOutput
-from texttools.internals.operator_utils import OperatorUtils
-from texttools.internals.prompt_loader import PromptLoader
-from texttools.internals.exceptions import (
-    TextToolsError,
-    LLMError,
-    ValidationError,
-    PromptError,
-)
+from ..engine import OperatorUtils, PromptLoader
+from ..exceptions import LLMError, PromptError, TextToolsError, ValidationError
+from ..internal_models import OperatorOutput
 
 # Base Model type for output models
 T = TypeVar("T", bound=BaseModel)