hamtaa-texttools 1.1.16__py3-none-any.whl → 1.2.0__py3-none-any.whl

texttools/core/operators/sync_operator.py

@@ -0,0 +1,192 @@
+from collections.abc import Callable
+from typing import Any, Type, TypeVar
+
+from openai import OpenAI
+from pydantic import BaseModel
+
+from ..engine import OperatorUtils, PromptLoader
+from ..exceptions import LLMError, PromptError, TextToolsError, ValidationError
+from ..internal_models import OperatorOutput
+
+# Base Model type for output models
+T = TypeVar("T", bound=BaseModel)
+
+
+class Operator:
+    """
+    Core engine for running text-processing operations with an LLM.
+    """
+
+    def __init__(self, client: OpenAI, model: str):
+        self._client = client
+        self._model = model
+
+    def _analyze_completion(self, analyze_message: list[dict[str, str]]) -> str:
+        try:
+            completion = self._client.chat.completions.create(
+                model=self._model,
+                messages=analyze_message,
+            )
+
+            if not completion.choices:
+                raise LLMError("No choices returned from LLM")
+
+            analysis = completion.choices[0].message.content.strip()
+
+            if not analysis:
+                raise LLMError("Empty analysis response")
+
+            return analysis
+
+        except Exception as e:
+            if isinstance(e, (PromptError, LLMError)):
+                raise
+            raise LLMError(f"Analysis failed: {e}")
+
+    def _parse_completion(
+        self,
+        main_message: list[dict[str, str]],
+        output_model: Type[T],
+        temperature: float,
+        logprobs: bool,
+        top_logprobs: int,
+        priority: int | None,
+    ) -> tuple[T, Any]:
+        """
+        Parses a chat completion using OpenAI's structured output format.
+        Returns both the parsed Any and the raw completion for logprobs.
+        """
+        try:
+            request_kwargs = {
+                "model": self._model,
+                "messages": main_message,
+                "response_format": output_model,
+                "temperature": temperature,
+            }
+
+            if logprobs:
+                request_kwargs["logprobs"] = True
+                request_kwargs["top_logprobs"] = top_logprobs
+
+            if priority is not None:
+                request_kwargs["extra_body"] = {"priority": priority}
+
+            completion = self._client.beta.chat.completions.parse(**request_kwargs)
+
+            if not completion.choices:
+                raise LLMError("No choices returned from LLM")
+
+            parsed = completion.choices[0].message.parsed
+
+            if not parsed:
+                raise LLMError("Failed to parse LLM response")
+
+            return parsed, completion
+
+        except Exception as e:
+            if isinstance(e, LLMError):
+                raise
+            raise LLMError(f"Completion failed: {e}")
+
+    def run(
+        self,
+        # User parameters
+        text: str,
+        with_analysis: bool,
+        output_lang: str | None,
+        user_prompt: str | None,
+        temperature: float,
+        logprobs: bool,
+        top_logprobs: int,
+        validator: Callable[[Any], bool] | None,
+        max_validation_retries: int | None,
+        priority: int | None,
+        # Internal parameters
+        tool_name: str,
+        output_model: Type[T],
+        mode: str | None,
+        **extra_kwargs,
+    ) -> OperatorOutput:
+        """
+        Execute the LLM pipeline with the given input text.
+        """
+        try:
+            prompt_loader = PromptLoader()
+            prompt_configs = prompt_loader.load(
+                prompt_file=tool_name + ".yaml",
+                text=text.strip(),
+                mode=mode,
+                **extra_kwargs,
+            )
+
+            analysis: str | None = None
+
+            if with_analysis:
+                analyze_message = OperatorUtils.build_message(
+                    prompt_configs["analyze_template"]
+                )
+                analysis = self._analyze_completion(analyze_message)
+
+            main_message = OperatorUtils.build_message(
+                OperatorUtils.build_main_prompt(
+                    prompt_configs["main_template"], analysis, output_lang, user_prompt
+                )
+            )
+
+            parsed, completion = self._parse_completion(
+                main_message,
+                output_model,
+                temperature,
+                logprobs,
+                top_logprobs,
+                priority,
+            )
+
+            # Retry logic if validation fails
+            if validator and not validator(parsed.result):
+                if (
+                    not isinstance(max_validation_retries, int)
+                    or max_validation_retries < 1
+                ):
+                    raise ValueError("max_validation_retries should be a positive int")
+
+                succeeded = False
+                for _ in range(max_validation_retries):
+                    # Generate a new temperature to retry
+                    retry_temperature = OperatorUtils.get_retry_temp(temperature)
+
+                    try:
+                        parsed, completion = self._parse_completion(
+                            main_message,
+                            output_model,
+                            retry_temperature,
+                            logprobs,
+                            top_logprobs,
+                            priority=priority,
+                        )
+
+                        # Check if retry was successful
+                        if validator(parsed.result):
+                            succeeded = True
+                            break
+
+                    except LLMError:
+                        pass
+
+                if not succeeded:
+                    raise ValidationError("Validation failed after all retries")
+
+            operator_output = OperatorOutput(
+                result=parsed.result,
+                analysis=analysis if with_analysis else None,
+                logprobs=OperatorUtils.extract_logprobs(completion)
+                if logprobs
+                else None,
+            )
+
+            return operator_output
+
+        except (PromptError, LLMError, ValidationError):
+            raise
+        except Exception as e:
+            raise TextToolsError(f"Unexpected error in operator: {e}")

texttools/models.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from pydantic import BaseModel
+
+
+class ToolOutputMetadata(BaseModel):
+    tool_name: str
+    processed_at: datetime = datetime.now()
+    execution_time: float | None = None
+
+
+class ToolOutput(BaseModel):
+    result: Any = None
+    analysis: str | None = None
+    logprobs: list[dict[str, Any]] | None = None
+    errors: list[str] = []
+    metadata: ToolOutputMetadata | None = None
+
+    def __repr__(self) -> str:
+        return f"ToolOutput({self.model_dump_json(indent=2)})"
+
+
+class Node:
+    def __init__(self, name: str, description: str, level: int, parent: Node | None):
+        self.name = name
+        self.description = description
+        self.level = level
+        self.parent = parent
+        self.children = {}
+
+
+class CategoryTree:
+    def __init__(self):
+        self._root = Node(name="root", description="root", level=0, parent=None)
+        self._all_nodes = {"root": self._root}
+
+    def get_all_nodes(self) -> dict[str, Node]:
+        return self._all_nodes
+
+    def get_level_count(self) -> int:
+        return max(node.level for node in self._all_nodes.values())
+
+    def get_node(self, name: str) -> Node | None:
+        return self._all_nodes.get(name)
+
+    def add_node(
+        self,
+        name: str,
+        parent_name: str,
+        description: str | None = None,
+    ) -> None:
+        if self.get_node(name):
+            raise ValueError(f"Cannot add {name} category twice")
+
+        parent = self.get_node(parent_name)
+
+        if not parent:
+            raise ValueError(f"Parent category '{parent_name}' not found")
+
+        node_data = {
+            "name": name,
+            "description": description if description else "No description provided",
+            "level": parent.level + 1,
+            "parent": parent,
+        }
+
+        new_node = Node(**node_data)
+        parent.children[name] = new_node
+        self._all_nodes[name] = new_node
+
+    def remove_node(self, name: str) -> None:
+        if name == "root":
+            raise ValueError("Cannot remove the root node")
+
+        node = self.get_node(name)
+        if not node:
+            raise ValueError(f"Category: '{name}' not found")
+
+        for child_name in list(node.children.keys()):
+            self.remove_node(child_name)
+
+        if node.parent:
+            del node.parent.children[name]
+
+        del self._all_nodes[name]

texttools/prompts/categorize.yaml

@@ -1,77 +1,36 @@
-main_template:
-
-  category_list: |
-    You are an expert classification agent.  
-    You receive a list of categories.
-
-    Your task:
-    - Read all provided categories carefully.
-    - Consider the user query, intent, and task explanation.
-    - Select exactly one category name from the list that best matches the user’s intent.
-    - Return only the category name, nothing else.
-
-    Rules:
-    - Never invent categories that are not in the list.
-    - If multiple categories seem possible, choose the closest match based on the description and user intent.
-    - If descriptions are missing or empty, rely on the category name.
-    - If the correct answer cannot be determined with certainty, choose the most likely one.
-
-    Output format:
-    {{
-    "reason": "Explanation of why the input belongs to the category"
-    "result": "<category_name_only>"
-    }}
-
-    Available categories with their descriptions:
-    {category_list}
-
-    The text that has to be categorized:
-    {input}
-
-  category_tree: |
-    You are an expert classification agent.  
-    You receive a list of categories at the current level of a hierarchical category tree.  
-
-    Your task:
-    - Read all provided categories carefully.
-    - Consider the user query, intent, and task explanation.
-    - Select exactly one category name from the list that best matches the user’s intent.
-    - Return only the category name, nothing else.
-
-    Rules:
-    - Never invent categories that are not in the list.
-    - If multiple categories seem possible, choose the closest match based on the description and user intent.
-    - If descriptions are missing or empty, rely on the category name.
-    - If the correct answer cannot be determined with certainty, choose the most likely one.
-
-    Output format:
-    {{
-    "reason": "Explanation of why the input belongs to the category"
-    "result": "<category_name_only>"
-    }}
-
-    Available categories with their descriptions at this level:
-    {category_list}
-
-    Do not include category descriptions at all. Only write the raw category.
-
-    The text that has to be categorized:
-    {input}
-
-analyze_template:
-
-  category_list: |
-    We want to categorize the given text.
-    To improve categorization, we need an analysis of the text.
-    Analyze the given text and write its main idea and a short analysis of that.
-    Analysis should be very short.
-    Text:
-    {input}
-
-  category_tree: |
-    We want to categorize the given text.
-    To improve categorization, we need an analysis of the text.
-    Analyze the given text and write its main idea and a short analysis of that.
-    Analysis should be very short.
-    Text:
-    {input}
+main_template: |
+  You are an expert classification agent.  
+  You receive a list of categories.
+
+  Your task:
+  - Read all provided categories carefully.
+  - Consider the user query, intent, and task explanation.
+  - Select exactly one category name from the list that best matches the user’s intent.
+  - Return only the category name, nothing else.
+
+  Rules:
+  - Never invent categories that are not in the list.
+  - If multiple categories seem possible, choose the closest match based on the description and user intent.
+  - If descriptions are missing or empty, rely on the category name.
+  - If the correct answer cannot be determined with certainty, choose the most likely one.
+
+  Output format:
+  {{
+  "reason": "Explanation of why the input belongs to the category"
+  "result": "<category_name_only>"
+  }}
+
+  Available categories with their descriptions:
+  {category_list}
+
+  Here is the text:
+  {text}
+
+analyze_template: |
+  We want to categorize the given text.
+  To improve categorization, we need an analysis of the text.
+  Analyze the given text and write its main idea and a short analysis of that.
+  Analysis should be very short.
+
+  Here is the text:
+  {text}

texttools/prompts/check_fact.yaml

@@ -0,0 +1,24 @@
+main_template: |
+  You are an expert in determining whether a statement can be concluded from the source text or not.
+  You must return a boolean value: True or False.
+  Return True if the statement can be concluded from the source, and False otherwise.
+  
+  Respond only in JSON format (Output should be a boolean):
+  {{"result": True/False}}
+
+  The statement is:
+  {text}
+
+  The source text is:
+  {source_text}
+
+analyze_template: |
+  You should analyze a statement and a source text and provide a brief,
+  summarized analysis that could help in determining that can the statement
+  be concluded from the source or not.
+
+  The statement is:
+  {text}
+  
+  The source text is:
+  {source_text}

texttools/prompts/extract_entities.yaml

@@ -1,7 +1,8 @@
 main_template: |
   You are a Named Entity Recognition (NER) extractor.
-  Identify and extract all named entities (e.g., PER, ORG, LOC, DAT, etc.) from the given text.
+  Identify and extract {entities} from the given text.
   For each entity, provide its text and a clear type.
+
   Respond only in JSON format:
   {{
     "result": [
@@ -11,10 +12,13 @@ main_template: |
       }}
     ]
   }}
+
   Here is the text:
-  {input}
+  {text}
 
 analyze_template: |
   Read the following text and identify any proper nouns, key concepts, or specific mentions that might represent named entities.
   Provide a brief, summarized analysis that could help in categorizing these entities.
-  {input}
+  
+  Here is the text:
+  {text}

texttools/prompts/extract_keywords.yaml

@@ -3,16 +3,19 @@ main_template:
   auto: |
     You are an expert keyword extractor.
     Extract the most relevant keywords from the given text.
+
     Guidelines:
     - Keywords must represent the main concepts of the text.  
     - If two words have overlapping meanings, choose only one.  
     - Do not include generic or unrelated words.  
     - Keywords must be single, self-contained words (no phrases).
     - Output between 3 and 7 keywords based on the input length.
-    - Respond only in JSON format:
+
+    Respond only in JSON format:
     {{"result": ["keyword1", "keyword2", etc.]}}
+
     Here is the text:
-    {input}
+    {text}
 
   threshold: |
     You are an expert keyword extractor specialized in fine-grained concept identification.
@@ -29,10 +32,12 @@ main_template:
       - Short texts (a few sentences): 3 keywords
       - Medium texts (1–4 paragraphs): 4–5 keywords
       - Long texts (more than 4 paragraphs): 6–7 keywords
-    - Respond only in JSON format:
+    
+    Respond only in JSON format:
     {{"result": ["keyword1", "keyword2", etc.]}}
+
     Here is the text:
-    {input}
+    {text}
 
   count: |
     You are an expert keyword extractor with precise output requirements.
@@ -45,24 +50,31 @@ main_template:
     - If the text doesn't contain enough distinct keywords, include the most relevant ones even if some are less specific.
     - Keywords must be single words (no multi-word expressions).
     - Order keywords by relevance (most relevant first).
-    - Respond only in JSON format:
+    
+    Respond only in JSON format:
     {{"result": ["keyword1", "keyword2", "keyword3", ...]}}
     
     Here is the text:
-    {input}
+    {text}
 
 analyze_template:
   auto: |
     Analyze the following text to identify its main topics, concepts, and important terms.
     Provide a concise summary of your findings that will help in extracting relevant keywords.
-    {input}
+    
+    Here is the text:
+    {text}
 
   threshold: |
     Analyze the following text to identify its main topics, concepts, and important terms.
     Provide a concise summary of your findings that will help in extracting relevant keywords.
-    {input}
+    
+    Here is the text:
+    {text}
 
   count: |
     Analyze the following text to identify its main topics, concepts, and important terms.
     Provide a concise summary of your findings that will help in extracting relevant keywords.
-    {input}
+    
+    Here is the text:
+    {text}

texttools/prompts/is_question.yaml

@@ -1,14 +1,18 @@
 main_template: |
   You are a question detector.
   Determine that if the given text contains any question or not.
+
   Respond only in JSON format (Output should be a boolean):
   {{"result": True/False}}
+
   Here is the text:
-  {input}
+  {text}
 
 analyze_template: |
   We want to analyze this text snippet to see if it contains any question or request of some kind or not.
   Read the text, and reason about it being a request or not.
   Summerized, short answer.
-  {input}
+
+  Here is the text:
+  {text}
   

texttools/prompts/merge_questions.yaml

@@ -4,15 +4,18 @@ main_template:
     You are a language expert.
     I will give you a list of questions that are semantically similar.
     Your task is to merge them into one unified question.
+    
     Guidelines:
     - Preserves all the information and intent from the original questions.
     - Sounds natural, fluent, and concise.
     - Avoids redundancy or unnecessary repetition.
     - Does not omit any unique idea from the originals.
-    - Respond only in JSON format:
+    
+    Respond only in JSON format:
     {{"result": "string"}}
+
     Here is the questions:
-    {input}
+    {text}
 
   reason: |
     You are an AI assistant helping to unify semantically similar questions.
@@ -20,10 +23,12 @@ main_template:
     Then, write one merged question that combines all their content clearly and naturally, without redundancy.
     Step 1: Extract key ideas.
     Step 2: Write the final merged question.
+
     Respond only in JSON format:
     {{"result": "string"}}
+
     Here is the questions:
-    {input}
+    {text}
 
 analyze_template:
 
@@ -33,14 +38,16 @@ analyze_template:
     and the specific information they are seeking.
     Provide a brief, summarized understanding of the questions' meaning that
     will help in merging and rephrasing it accurately without changing its intent.        
+    
     Here is the question:
-    {input}
+    {text}
 
   reason: |
     Analyze the following questions to identify their exact wording, phrasing,
     and the literal meaning it conveys.
     Provide a brief, summarized analysis of their linguistic structure and current meaning,
     which will then be used to create a new question containing all of their contents.
+    
     Here is the question:
-    {input}
+    {text}
     

texttools/prompts/propositionize.yaml

@@ -0,0 +1,24 @@
+main_template: |
+  You are an expert data analyst specializing in Information Extraction.
+  Your task is to extract a list of "Atomic Propositions" from the provided text.
+
+  Definition of Atomic Proposition:
+  A single, self-contained statement of fact that is concise and verifiable.
+
+  Strict Guidelines:
+  1. Remove Meta-Data: STRICTLY EXCLUDE all citations, references, URLs, source attributions (e.g., "Source: makarem.ir"), and conversational fillers (e.g., "Based on the documents...", "In conclusion...").
+  2. Resolve Context: Replace pronouns ("it", "this", "they") with the specific nouns they refer to. Each proposition must make sense in isolation.
+  3. Preserve Logic: Keep conditions attached to their facts. Do not split a rule from its condition (e.g., "If X, then Y" should be one proposition).
+  4. No Redundancy: Do not extract summary statements that merely repeat facts already listed.
+
+  Extract the atomic propositions from the following text:
+  {text}
+
+analyze_template: |
+  We want to analyze this text snippet and think about where we can split sentence to atomic meaningful propositions.
+  An atomic proposition is a single, self-contained fact that is concise,
+  verifiable, and does not rely on external context.
+  You just have to think around the possible propositions in the text and how a proposition can be made.
+  
+  Here is the text:
+  {text}

texttools/prompts/rewrite.yaml

@@ -18,7 +18,7 @@ main_template:
     {{"result": "str"}}
 
     Anchor Text:
-    "{input}"
+    "{text}"
 
   negative: |
     You are an AI assistant designed to generate high-quality training data for semantic text embedding models.
@@ -35,7 +35,7 @@ main_template:
     {{"result": "str"}}
 
     Anchor Text:
-    "{input}"
+    "{text}"
 
   hard_negative: |
       You are an AI assistant designed to generate high-quality training data for semantic text embedding models.
@@ -52,12 +52,11 @@ main_template:
       - Make it Challenging: The difference should be subtle enough that it requires a deep understanding of the text to identify, not just a simple keyword mismatch.
       - Maintain Similar Length: The generated sentence should be of roughly the same length and level of detail as the Anchor.
 
-
       Respond only in JSON format:
       {{"result": "str"}}
 
       Anchor Text:
-      "{input}"
+      "{text}"
 
 
 analyze_template:
@@ -73,8 +72,8 @@ analyze_template:
     
     Your analysis should capture the ESSENTIAL MEANING that must be preserved in any paraphrase.
     
-    Text:
-    {input}
+    Here is the text:
+    {text}
 
   negative: |
     Analyze the following text to identify its SPECIFIC TOPIC and DOMAIN for creating a high-quality NEGATIVE sample.
@@ -87,8 +86,8 @@ analyze_template:
     
     The goal is to find topics that are in the same domain but semantically unrelated to this specific text.
     
-    Text:
-    {input}
+    Here is the text:
+    {text}
 
   hard_negative: |
     Analyze this text to identify EXACTLY ONE ELEMENT that can be changed to create a hard-negative sample.
@@ -106,6 +105,6 @@ analyze_template:
     - Sentence structure
     - 80-90% of the vocabulary
     
-    Text:
-    {input}
+    Here is the text:
+    {text}