hamtaa-texttools 0.1.43__py3-none-any.whl → 1.0.0__py3-none-any.whl

hamtaa_texttools-1.0.0.dist-info/METADATA

@@ -0,0 +1,129 @@
+Metadata-Version: 2.4
+Name: hamtaa-texttools
+Version: 1.0.0
+Summary: TextTools is a high-level NLP toolkit built on top of modern LLMs.
+Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, MoosaviNejad <erfanmoosavi84@gmail.com>
+License: MIT License
+        
+        Copyright (c) 2025 Hamtaa
+        
+        Permission is hereby granted, free of charge, to any person obtaining a copy
+        of this software and associated documentation files (the "Software"), to deal
+        in the Software without restriction, including without limitation the rights
+        to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+        copies of the Software, and to permit persons to whom the Software is
+        furnished to do so, subject to the following conditions:
+        
+        The above copyright notice and this permission notice shall be included in all
+        copies or substantial portions of the Software.
+        
+        THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+        IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+        FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+        AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+        LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+        OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+        SOFTWARE.
+Keywords: nlp,llm,text-processing,openai
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: openai==1.97.1
+Requires-Dist: PyYAML>=6.0
+Dynamic: license-file
+
+# TextTools
+
+## 📌 Overview
+
+**TextTools** is a high-level **NLP toolkit** built on top of modern **LLMs**.  
+It provides ready-to-use utilities for **translation, question detection, keyword extraction, categorization, NER extractor, and more** — designed to help you integrate AI-powered text processing into your applications with minimal effort.
+
+---
+
+## ✨ Features
+
+TextTools provides a rich collection of high-level NLP utilities built on top of LLMs.  
+Each tool is designed to work out-of-the-box with structured outputs (JSON / Pydantic).
+
+- **Categorizer** → Zero-finetuning text categorization for fast, scalable classification.  
+- **Keyword Extractor** → Identify the most important keywords in a text.  
+- **Question Merger** → Merge the provided questions, preserving all the main points 
+- **NER (Named Entity Recognition) Extractor** → Extract people, places, organizations, and other entities.  
+- **Question Detector** → Determine whether a text is a question or not.  
+- **Question Generator From Text** → Generate high-quality, context-relevant questions from provided text.
+- **Question Generator From Subject** → Generate high-quality, context-relevant questions from a subject.
+- **Rewriter** → Rewrite text while preserving meaning or without it.
+- **Summarizer** → Condense long passages into clear, structured summaries. 
+- **Translator** → Translate text across multiple languages, with support for custom rules.
+
+---
+
+## 🔍 `with_analysis` Mode
+
+The `with_analysis=True` flag enhances the tool's output by providing a detailed reasoning chain behind its result. This is valuable for debugging, improving prompts, or understanding model behavior.
+
+**Please be aware:** This feature works by making an additional LLM API call for each tool invocation, which will **effectively double your token usage** for that operation.
+
+---
+
+## 🚀 Installation
+
+Install the latest release via PyPI:
+
+```bash
+pip install -U hamta-texttools
+```
+
+---
+
+## ⚡ Quick Start
+
+```python
+from openai import OpenAI
+
+from texttools import TheTool
+
+# Create your OpenAI client
+client = OpenAI(base_url = "your_url", API_KEY = "your_api_key")
+
+# Specify the model
+model = "gpt-4o-mini"
+
+# Create an instance of TheTool
+# ⚠️ Note: Enabling `with_analysis=True` provides deeper insights but incurs additional LLM calls and token usage.
+the_tool = TheTool(client = client, model = model, with_analysis = True)
+
+# Example: Question Detection
+print(the_tool.detect_question("Is this project open source?")["result"])
+# Output: True
+
+# Example: Translation
+print(the_tool.translate("سلام، حالت چطوره؟")["result"])
+# Output: "Hi! How are you?"
+```
+
+---
+
+## 📚 Use Cases
+
+Use **TextTools** when you need to:
+
+- 🔍 **Classify** large datasets quickly without model training  
+- 🌍 **Translate** and process multilingual corpora with ease  
+- 🧩 **Integrate** LLMs into production pipelines (structured outputs)  
+- 📊 **Analyze** large text collections using embeddings and categorization  
+- ⚙️ **Automate** common text-processing tasks without reinventing the wheel  
+
+---
+
+## 🤝 Contributing
+
+Contributions are welcome!  
+Feel free to **open issues, suggest new features, or submit pull requests**.  
+
+---
+
+## License
+
+This project is licensed under the MIT License - see the [LICENSE](LICENSE) file for details.

hamtaa_texttools-1.0.0.dist-info/RECORD

@@ -0,0 +1,17 @@
+hamtaa_texttools-1.0.0.dist-info/licenses/LICENSE,sha256=TJch8KUnfKaKJFkaRqgtghB7rtprhaHyGirYKr90U4o,1062
+texttools/__init__.py,sha256=DEPDeR8rKRye57x9kq00Adq9GOLFkmaWRq9sGBNQZ_c,241
+texttools/formatters/base_formatter.py,sha256=sUrISJcczTLDPMiMETG-kyfZ64u0NubFpT3mjEQBskk,1147
+texttools/formatters/user_merge_formatter/user_merge_formatter.py,sha256=R-e64Gwq6jARcpsnPYsgNIX7eqFDi0BtfiZOATvwxqo,1692
+texttools/tools/__init__.py,sha256=Gzqlobmbgd5wOvy27JYPKB74MFtqDgFy6LwlRygN240,53
+texttools/tools/operator.py,sha256=g3ZC5OSxG_oZQkkMbfzc8uUvw0FNvehNB5jPPY26KEg,7972
+texttools/tools/output_models.py,sha256=EdMGvPEp0k8l9Ps48Arw7GMcXSmdRLPrvAhaYnVqGj8,1099
+texttools/tools/prompt_loader.py,sha256=zrCgLNGkFV60u6b7CN4dNcml4cGLrC2ei0WcMfD28Bc,2817
+texttools/tools/the_tool.py,sha256=lEMVpqhJvPqVzSWx8NlmYV7jqZ1ul3IqJ9nHJLjz0bw,9653
+texttools/utils/__init__.py,sha256=XL_cVGbe8wKf8HQh_Q1JEZgGOlmpLijPoHNvzi1aYnc,167
+texttools/utils/batch_manager/__init__.py,sha256=WcnujCd_5XotN6emVCfDaO_lMpyk8EwJYcFgNRks5q0,139
+texttools/utils/batch_manager/batch_manager.py,sha256=N7dg1bE0QpGYjHtM0E9DWtXErZR_z0byls9d8RQdUbs,9104
+texttools/utils/batch_manager/batch_runner.py,sha256=3dhzmHrvCKqQVTtxeBIiUhCyRwKiQp_WmWqGX2WTG-o,7602
+hamtaa_texttools-1.0.0.dist-info/METADATA,sha256=kGjkvffaWgdZZletnEyj6eH-PCgHADXyMSW8YF1dclA,5114
+hamtaa_texttools-1.0.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+hamtaa_texttools-1.0.0.dist-info/top_level.txt,sha256=5Mh0jIxxZ5rOXHGJ6Mp-JPKviywwN0MYuH0xk5bEWqE,10
+hamtaa_texttools-1.0.0.dist-info/RECORD,,

hamtaa_texttools-1.0.0.dist-info/licenses/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Hamtaa
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

texttools/__init__.py

@@ -1,26 +1,9 @@
-from texttools.batch_manager import BatchJobRunner, SimpleBatchManager
-from texttools.handlers import (
-    NoOpResultHandler,
-    PrintResultHandler,
-    ResultHandler,
-    SaveToFileResultHandler,
-)
-from texttools.tools.categorizer.encoder_model.encoder_vectorizer import (
-    EmbeddingCategorizer,
-)
-from texttools.tools.categorizer.llm.openai_categorizer import LLMCategorizer
-from texttools.tools.question_detector.llm_detector import LLMQuestionDetector
-from texttools.tools.summarizer import LLMSummarizer
+from .tools.the_tool import TheTool
+from .utils.batch_manager.batch_manager import SimpleBatchManager
+from .utils.batch_manager.batch_runner import BatchJobRunner
 
 __all__ = [
-    "LLMQuestionDetector",
-    "NoOpResultHandler",
-    "PrintResultHandler",
-    "ResultHandler",
-    "SaveToFileResultHandler",
-    "EmbeddingCategorizer",
-    "LLMCategorizer",
+    "TheTool",
     "SimpleBatchManager",
     "BatchJobRunner",
-    "LLMSummarizer",
 ]

texttools/formatters/base_formatter.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class BaseFormatter(ABC):
+    """
+    Adapter to convert a conversation into a specific LLM API's input format.
+
+    Concrete implementations transform standardized messages (e.g., list[dict]) into the
+    exact payload required by a provider (e.g., OpenAI's message list, a single string, etc.).
+    """
+
+    @abstractmethod
+    def format(
+        self,
+        messages: Any,
+    ) -> Any:
+        """
+        Transform the input messages into a provider-specific payload.
+
+        Args:
+            messages: The input conversation. While often a list of dicts with
+                      'role' and 'content' keys, the exact type and structure may vary
+                      by implementation.
+
+        Returns:
+            A payload in the format expected by the target LLM API. This could be:
+            - A list of role-content dictionaries (e.g., for OpenAI)
+            - A single formatted string (e.g., for completion-style APIs)
+            - A complex dictionary with additional parameters
+            - Any other provider-specific data structure
+        """
+        pass

texttools/formatters/user_merge_formatter/user_merge_formatter.py

@@ -0,0 +1,47 @@
+from texttools.formatters.base_formatter import BaseFormatter
+
+
+class UserMergeFormatter(BaseFormatter):
+    """
+    Merges consecutive user messages into a single message, separated by newlines.
+
+    This is useful for condensing a multi-turn user input into a single coherent
+    message for the LLM. Assistant and system messages are left unchanged and
+    act as separators between user message groups.
+
+    Raises:
+        ValueError: If the input messages have invalid structure or roles.
+    """
+
+    def _validate_input(self, messages: list[dict[str, str]]):
+        valid_keys = {"role", "content"}
+        valid_roles = {"user", "assistant"}
+
+        for message in messages:
+            # Validate keys
+            if set(message.keys()) != valid_keys:
+                raise ValueError(
+                    f"Message dict keys must be exactly {valid_keys}, got {set(message.keys())}"
+                )
+            # Validate roles
+            role = message["role"]
+            if role != "system" and role not in valid_roles:
+                raise ValueError(f"Unexpected role: {role}")
+
+    def format(self, messages: list[dict[str, str]]) -> list[dict[str, str]]:
+        self._validate_input(messages)
+
+        merged: list[dict[str, str]] = []
+
+        for message in messages:
+            role, content = message["role"], message["content"].strip()
+
+            # Merge with previous user turn
+            if merged and role == "user" and merged[-1]["role"] == "user":
+                merged[-1]["content"] += "\n" + content
+
+            # Otherwise, start a new turn
+            else:
+                merged.append({"role": role, "content": content})
+
+        return merged

texttools/tools/__init__.py

@@ -1,33 +1,3 @@
-from .categorizer import EmbeddingCategorizer, GemmaCategorizer, LLMCategorizer
-from .keyword_extractor import GemmaKeywordExtractor
-from .ner import GemmaNERExtractor
-from .question_detector import GemmaQuestionDetector, LLMQuestionDetector
-from .question_generator import GemmaQuestionGenerator
-from .reranker import GemmaReranker, GemmaScorer, GemmaSorter
-from .rewriter import GemmaQuestionRewriter, RewriteMode
-from .merger import GemmaQuestionMerger, MergingMode
-from .subject_to_question import GemmaQuestionGeneratorFromSubject
-from .summarizer import GemmaSummarizer, LLMSummarizer
-from .translator import GemmaTranslator
+from .the_tool import TheTool
 
-__all__ = [
-    "EmbeddingCategorizer",
-    "GemmaCategorizer",
-    "LLMCategorizer",
-    "GemmaTranslator",
-    "GemmaSummarizer",
-    "LLMSummarizer",
-    "GemmaNERExtractor",
-    "GemmaQuestionDetector",
-    "LLMQuestionDetector",
-    "GemmaQuestionGenerator",
-    "GemmaScorer",
-    "GemmaSorter",
-    "GemmaReranker",
-    "GemmaQuestionRewriter",
-    "RewriteMode",
-    "GemmaKeywordExtractor",
-    "GemmaQuestionGeneratorFromSubject",
-    "GemmaQuestionMerger",
-    "MergingMode",
-]
+__all__ = ["TheTool"]

texttools/tools/operator.py

@@ -0,0 +1,236 @@
+from __future__ import annotations
+
+from typing import Any, TypeVar, Type, Literal
+import json
+
+from openai import OpenAI
+from pydantic import BaseModel
+
+from texttools.formatters.user_merge_formatter.user_merge_formatter import (
+    UserMergeFormatter,
+)
+from texttools.tools.prompt_loader import PromptLoader
+
+# Base Model type for output models
+T = TypeVar("T", bound=BaseModel)
+
+
+class Operator:
+    """
+    Core engine for running text-processing operations with an LLM.
+
+    It wires together:
+    - `PromptLoader` → loads YAML prompt templates.
+    - `UserMergeFormatter` → applies formatting to messages (e.g., merging).
+    - OpenAI client → executes completions/parsed completions.
+
+    Workflow inside `run()`:
+    1. Load prompt templates (`main_template` [+ `analyze_template` if enabled]).
+    2. Optionally generate an "analysis" step via `_analyze()`.
+    3. Build messages for the LLM.
+    4. Call `.beta.chat.completions.parse()` to parse the result into the
+       configured `OUTPUT_MODEL` (a Pydantic schema).
+    5. Return results as a dict (always `{"result": ...}`, plus `analysis`
+       if analysis was enabled).
+
+    Attributes configured dynamically by `TheTool`:
+    - PROMPT_FILE: str → YAML filename
+    - OUTPUT_MODEL: Pydantic model class
+    - WITH_ANALYSIS: bool → whether to run an analysis phase first
+    - USE_MODES: bool → whether to select prompts by mode
+    - MODE: str → which mode to use if modes are enabled
+    - RESP_FORMAT: str → "vllm" or "parse"
+    """
+
+    PROMPT_FILE: str
+    OUTPUT_MODEL: Type[T]
+    WITH_ANALYSIS: bool = False
+    USE_MODES: bool
+    MODE: str = ""
+    RESP_FORMAT: Literal["vllm", "parse"] = "vllm"
+
+    def __init__(
+        self,
+        client: OpenAI,
+        *,
+        model: str,
+        temperature: float = 0.0,
+        **client_kwargs: Any,
+    ):
+        self.client: OpenAI = client
+        self.model = model
+        self.prompt_loader = PromptLoader()
+        self.formatter = UserMergeFormatter()
+        self.temperature = temperature
+        self.client_kwargs = client_kwargs
+
+    def _build_user_message(self, prompt: str) -> dict[str, str]:
+        return {"role": "user", "content": prompt}
+
+    def _apply_formatter(self, messages: list[dict[str, str]]) -> list[dict[str, str]]:
+        return self.formatter.format(messages)
+
+    def _analysis_completion(self, analyze_message: list[dict[str, str]]) -> str:
+        try:
+            completion = self.client.chat.completions.create(
+                model=self.model,
+                messages=analyze_message,
+                temperature=self.temperature,
+                **self.client_kwargs,
+            )
+            analysis = completion.choices[0].message.content.strip()
+            return analysis
+
+        except Exception as e:
+            print(f"[ERROR] Analysis failed: {e}")
+            raise
+
+    def _analyze(self) -> str:
+        analyze_prompt = self.prompt_configs["analyze_template"]
+        analyze_message = [self._build_user_message(analyze_prompt)]
+        analysis = self._analysis_completion(analyze_message)
+
+        return analysis
+
+    def _build_main_message(self) -> list[dict[str, str]]:
+        main_prompt = self.prompt_configs["main_template"]
+        main_message = self._build_user_message(main_prompt)
+
+        return main_message
+
+    def _parse_completion(self, message: list[dict[str, str]]) -> T:
+        try:
+            completion = self.client.beta.chat.completions.parse(
+                model=self.model,
+                messages=message,
+                response_format=self.OUTPUT_MODEL,
+                temperature=self.temperature,
+                **self.client_kwargs,
+            )
+            parsed = completion.choices[0].message.parsed
+            return parsed
+
+        except Exception as e:
+            print(f"[ERROR] Failed to parse completion: {e}")
+            raise
+
+    def _clean_json_response(self, response: str) -> str:
+        """
+        Clean JSON response by removing code block markers and whitespace.
+        Handles cases like:
+        - ```json{"result": "value"}```
+        - ```{"result": "value"}```
+        """
+        # Remove code block markers
+        cleaned = response.strip()
+
+        # Remove ```json and ``` markers
+        if cleaned.startswith("```json"):
+            cleaned = cleaned[7:]  # Remove ```json
+        elif cleaned.startswith("```"):
+            cleaned = cleaned[3:]  # Remove ```
+
+        # Remove trailing ``` or '''
+        if cleaned.endswith("```"):
+            cleaned = cleaned[:-3]
+
+        return cleaned.strip()
+
+    def _convert_to_output_model(self, response_string: str) -> T:
+        """
+        Convert a JSON response string to output model.
+
+        Args:
+            response_string: The JSON string (may contain code block markers)
+            output_model: Your Pydantic output model class (e.g., StrOutput, ListStrOutput)
+
+        Returns:
+            Instance of your output model
+        """
+        try:
+            # Clean the response string
+            cleaned_json = self._clean_json_response(response_string)
+
+            # Convert string to Python dictionary
+            response_dict = json.loads(cleaned_json)
+
+            # Convert dictionary to output model
+            return self.OUTPUT_MODEL(**response_dict)
+
+        except json.JSONDecodeError as e:
+            raise ValueError(
+                f"Failed to parse JSON response: {e}\nResponse: {response_string}"
+            )
+        except Exception as e:
+            raise ValueError(f"Failed to convert to output model: {e}")
+
+    def _vllm_completion(self, message: list[dict[str, str]]) -> T:
+        try:
+            json_schema = self.OUTPUT_MODEL.model_json_schema()
+            completion = self.client.chat.completions.create(
+                model=self.model,
+                messages=message,
+                extra_body={"guided_json": json_schema},
+                temperature=self.temperature,
+                **self.client_kwargs,
+            )
+            response = completion.choices[0].message.content
+
+            # Convert the string response to output model
+            parsed_response = self._convert_to_output_model(response)
+
+            return parsed_response
+
+        except Exception as e:
+            print(f"[ERROR] Failed to get vLLM structured output: {e}")
+            raise
+
+    def run(self, input_text: str, **extra_kwargs) -> dict[str, Any]:
+        """
+        Execute the LLM pipeline with the given input text.
+
+        Args:
+            input_text: The text to process (will be stripped of whitespace)
+            **extra_kwargs: Additional variables to inject into prompt templates
+
+        Returns:
+            Dictionary containing the parsed result and optional analysis
+        """
+        try:
+            cleaned_text = input_text.strip()
+
+            self.prompt_configs = self.prompt_loader.load_prompts(
+                self.PROMPT_FILE,
+                self.USE_MODES,
+                self.MODE,
+                cleaned_text,
+                **extra_kwargs,
+            )
+
+            messages: list[dict[str, str]] = []
+
+            if self.WITH_ANALYSIS:
+                analysis = self._analyze()
+                messages.append(
+                    self._build_user_message(f"Based on this analysis: {analysis}")
+                )
+
+            messages.append(self._build_main_message())
+            messages = self.formatter.format(messages)
+
+            if self.RESP_FORMAT == "vllm":
+                parsed = self._vllm_completion(messages)
+            elif self.RESP_FORMAT == "parse":
+                parsed = self._parse_completion(messages)
+
+            results = {"result": parsed.result}
+
+            if self.WITH_ANALYSIS:
+                results["analysis"] = analysis
+
+            return results
+
+        except Exception as e:
+            # Print error clearly and exit
+            print(f"[ERROR] Operation failed: {e}")
+            exit(1)

texttools/tools/output_models.py

@@ -0,0 +1,54 @@
+from typing import Literal
+
+from pydantic import BaseModel
+
+
+class StrOutput(BaseModel):
+    """
+    Output model for a single string result.
+    """
+
+    result: str
+
+
+class ListStrOutput(BaseModel):
+    """
+    Output model for a list of strings result.
+    """
+
+    result: list[str]
+
+
+class ListDictStrStrOutput(BaseModel):
+    """
+    Output model for a list of dictionaries with string key-value pairs.
+    """
+
+    result: list[dict[str, str]]
+
+
+class ReasonListStrOutput(BaseModel):
+    """
+    Output model containing a reasoning string followed by a list of strings.
+    """
+
+    reason: str
+    result: list[str]
+
+
+class CategorizerOutput(BaseModel):
+    """
+    Output model for categorization with reasoning and a predefined category result.
+    """
+
+    reason: str
+    result: Literal[
+        "باورهای دینی",
+        "اخلاق اسلامی",
+        "احکام و فقه",
+        "تاریخ اسلام و شخصیت ها",
+        "منابع دینی",
+        "دین و جامعه/سیاست",
+        "عرفان و معنویت",
+        "هیچکدام",
+    ]