hamtaa-texttools 1.1.16__py3-none-any.whl → 1.2.0__py3-none-any.whl

texttools/tools/internals/models.py

@@ -1,183 +0,0 @@
-from datetime import datetime
-from typing import Type, Any, Literal
-
-from pydantic import BaseModel, Field, create_model
-
-
-class ToolOutput(BaseModel):
-    result: Any = None
-    analysis: str = ""
-    logprobs: list[dict[str, Any]] = []
-    process: str = ""
-    processed_at: datetime = datetime.now()
-    execution_time: float = -1.0
-    errors: list[str] = []
-
-    def __repr__(self) -> str:
-        return f"ToolOutput(process='{self.process}', result_type='{type(self.result)}', result='{self.result}', analysis='{self.analysis}', logprobs='{self.logprobs}', errors='{self.errors}', processed_at='{self.processed_at}', execution_time='{self.execution_time}'"
-
-
-class StrOutput(BaseModel):
-    result: str = Field(..., description="The output string")
-
-
-class BoolOutput(BaseModel):
-    result: bool = Field(
-        ..., description="Boolean indicating the output state", example=True
-    )
-
-
-class ListStrOutput(BaseModel):
-    result: list[str] = Field(
-        ..., description="The output list of strings", example=["text_1", "text_2"]
-    )
-
-
-class ListDictStrStrOutput(BaseModel):
-    result: list[dict[str, str]] = Field(
-        ...,
-        description="List of dictionaries containing string key-value pairs",
-        example=[{"text": "Mohammad", "type": "PER"}],
-    )
-
-
-class ReasonListStrOutput(BaseModel):
-    reason: str = Field(..., description="Thinking process that led to the output")
-    result: list[str] = Field(..., description="The output list of strings")
-
-
-class Node(BaseModel):
-    node_id: int
-    name: str
-    level: int
-    parent_id: int | None
-    description: str = "No description provided"
-
-
-class CategoryTree:
-    def __init__(self, tree_name):
-        self.root = Node(node_id=0, name=tree_name, level=0, parent_id=None)
-        self.all_nodes: list[Node] = [self.root]
-        self.new_id = 1
-
-    def add_node(
-        self,
-        node_name: str,
-        parent_name: str | None = None,
-        description: str | None = None,
-    ) -> None:
-        if self.find_node(node_name):
-            raise ValueError(f"{node_name} has been chosen for another category before")
-
-        if parent_name:
-            parent_node = self.find_node(parent_name)
-            if parent_node is None:
-                raise ValueError(f"Parent category '{parent_name}' not found")
-            parent_id = parent_node.node_id
-            level = parent_node.level + 1
-        else:
-            level = 1
-            parent_id = 0
-
-        node_data = {
-            "node_id": self.new_id,
-            "name": node_name,
-            "level": level,
-            "parent_id": parent_id,
-        }
-
-        if description is not None:
-            node_data["description"] = description
-
-        self.all_nodes.append(Node(**node_data))
-        self.new_id += 1
-
-    def get_nodes(self) -> list[Node]:
-        return self.all_nodes
-
-    def get_level_count(self) -> int:
-        return max([item.level for item in self.all_nodes])
-
-    def find_node(self, identifier: int | str) -> Node | None:
-        if isinstance(identifier, str):
-            for node in self.get_nodes():
-                if node.name == identifier:
-                    return node
-            return None
-        elif isinstance(identifier, int):
-            for node in self.get_nodes():
-                if node.node_id == identifier:
-                    return node
-            return None
-        else:
-            return None
-
-    def find_children(self, parent_node: Node) -> list[Node] | None:
-        children = [
-            node for node in self.get_nodes() if parent_node.node_id == node.parent_id
-        ]
-        return children if children else None
-
-    def remove_node(self, identifier: int | str) -> None:
-        node = self.find_node(identifier)
-
-        if node is not None:
-            # Remove node's children recursively
-            children = self.find_children(node)
-
-            # Ending condition
-            if children is None:
-                self.all_nodes.remove(node)
-                return
-
-            for child in children:
-                self.remove_node(child.name)
-
-            # Remove the node from tree
-            self.all_nodes.remove(node)
-        else:
-            raise ValueError(f"Node with identifier: '{identifier}' not found.")
-
-    def dump_tree(self) -> dict:
-        def build_dict(node: Node) -> dict:
-            children = [
-                build_dict(child)
-                for child in self.all_nodes
-                if child.parent_id == node.node_id
-            ]
-            return {
-                "node_id": node.node_id,
-                "name": node.name,
-                "level": node.level,
-                "parent_id": node.parent_id,
-                "children": children,
-            }
-
-        return {"category_tree": build_dict(self.root)["children"]}
-
-
-# This function is needed to create CategorizerOutput with dynamic categories
-def create_dynamic_model(allowed_values: list[str]) -> Type[BaseModel]:
-    literal_type = Literal[*allowed_values]
-
-    CategorizerOutput = create_model(
-        "CategorizerOutput",
-        reason=(
-            str,
-            Field(
-                ..., description="Explanation of why the input belongs to the category"
-            ),
-        ),
-        result=(literal_type, Field(..., description="Predicted category label")),
-    )
-
-    return CategorizerOutput
-
-
-class Entity(BaseModel):
-    text: str = Field(description="The exact text of the entity")
-    entity_type: str = Field(description="The type of the entity")
-
-
-class EntityDetectorOutput(BaseModel):
-    result: list[Entity] = Field(description="List of all extracted entities")

texttools/tools/internals/operator_utils.py

@@ -1,54 +0,0 @@
-import re
-import math
-import random
-
-
-class OperatorUtils:
-    @staticmethod
-    def build_user_message(prompt: str) -> dict[str, str]:
-        return {"role": "user", "content": prompt}
-
-    @staticmethod
-    def extract_logprobs(completion: dict) -> list[dict]:
-        """
-        Extracts and filters token probabilities from completion logprobs.
-        Skips punctuation and structural tokens, returns cleaned probability data.
-        """
-        logprobs_data = []
-
-        ignore_pattern = re.compile(r'^(result|[\s\[\]\{\}",:]+)$')
-
-        for choice in completion.choices:
-            if not getattr(choice, "logprobs", None):
-                return []
-
-            for logprob_item in choice.logprobs.content:
-                if ignore_pattern.match(logprob_item.token):
-                    continue
-                token_entry = {
-                    "token": logprob_item.token,
-                    "prob": round(math.exp(logprob_item.logprob), 8),
-                    "top_alternatives": [],
-                }
-                for alt in logprob_item.top_logprobs:
-                    if ignore_pattern.match(alt.token):
-                        continue
-                    token_entry["top_alternatives"].append(
-                        {
-                            "token": alt.token,
-                            "prob": round(math.exp(alt.logprob), 8),
-                        }
-                    )
-                logprobs_data.append(token_entry)
-
-        return logprobs_data
-
-    @staticmethod
-    def get_retry_temp(base_temp: float) -> float:
-        """
-        Calculate temperature for retry attempts.
-        """
-        delta_temp = random.choice([-1, 1]) * random.uniform(0.1, 0.9)
-        new_temp = base_temp + delta_temp
-
-        return max(0.0, min(new_temp, 1.5))

texttools/tools/internals/prompt_loader.py

@@ -1,56 +0,0 @@
-from functools import lru_cache
-from pathlib import Path
-import yaml
-
-
-class PromptLoader:
-    """
-    Utility for loading and formatting YAML prompt templates.
-
-    Responsibilities:
-    - Load and parse YAML prompt definitions.
-    - Select the right template (by mode, if applicable).
-    - Inject variables (`{input}`, plus any extra kwargs) into the templates.
-    """
-
-    MAIN_TEMPLATE = "main_template"
-    ANALYZE_TEMPLATE = "analyze_template"
-
-    @staticmethod
-    def _build_format_args(text: str, **extra_kwargs) -> dict[str, str]:
-        # Base formatting args
-        format_args = {"input": text}
-        # Merge extras
-        format_args.update(extra_kwargs)
-        return format_args
-
-    # Use lru_cache to load each file once
-    @lru_cache(maxsize=32)
-    def _load_templates(self, prompt_file: str, mode: str | None) -> dict[str, str]:
-        """
-        Loads prompt templates from YAML file with optional mode selection.
-        """
-        base_dir = Path(__file__).parent.parent.parent / Path("prompts")
-        prompt_path = base_dir / prompt_file
-        data = yaml.safe_load(prompt_path.read_text(encoding="utf-8"))
-
-        return {
-            self.MAIN_TEMPLATE: data[self.MAIN_TEMPLATE][mode]
-            if mode
-            else data[self.MAIN_TEMPLATE],
-            self.ANALYZE_TEMPLATE: data.get(self.ANALYZE_TEMPLATE)[mode]
-            if mode
-            else data.get(self.ANALYZE_TEMPLATE),
-        }
-
-    def load(
-        self, prompt_file: str, text: str, mode: str, **extra_kwargs
-    ) -> dict[str, str]:
-        template_configs = self._load_templates(prompt_file, mode)
-        format_args = self._build_format_args(text, **extra_kwargs)
-
-        # Inject variables inside each template
-        for key in template_configs.keys():
-            template_configs[key] = template_configs[key].format(**format_args)
-
-        return template_configs

texttools/tools/internals/sync_operator.py

@@ -1,201 +0,0 @@
-from typing import Any, TypeVar, Type
-from collections.abc import Callable
-import logging
-
-from openai import OpenAI
-from pydantic import BaseModel
-
-from texttools.tools.internals.models import ToolOutput
-from texttools.tools.internals.operator_utils import OperatorUtils
-from texttools.tools.internals.formatters import Formatter
-from texttools.tools.internals.prompt_loader import PromptLoader
-
-# Base Model type for output models
-T = TypeVar("T", bound=BaseModel)
-
-logger = logging.getLogger("texttools.operator")
-
-
-class Operator:
-    """
-    Core engine for running text-processing operations with an LLM (Sync).
-
-    It wires together:
-    - `PromptLoader` → loads YAML prompt templates.
-    - `UserMergeFormatter` → applies formatting to messages (e.g., merging).
-    - OpenAI client → executes completions/parsed completions.
-    """
-
-    def __init__(self, client: OpenAI, model: str):
-        self._client = client
-        self._model = model
-
-    def _analyze(self, prompt_configs: dict[str, str], temperature: float) -> str:
-        """
-        Calls OpenAI API for analysis using the configured prompt template.
-        Returns the analyzed content as a string.
-        """
-        analyze_prompt = prompt_configs["analyze_template"]
-        analyze_message = [OperatorUtils.build_user_message(analyze_prompt)]
-        completion = self._client.chat.completions.create(
-            model=self._model,
-            messages=analyze_message,
-            temperature=temperature,
-        )
-        analysis = completion.choices[0].message.content.strip()
-        return analysis
-
-    def _parse_completion(
-        self,
-        message: list[dict[str, str]],
-        output_model: Type[T],
-        temperature: float,
-        logprobs: bool = False,
-        top_logprobs: int = 3,
-        priority: int | None = 0,
-    ) -> tuple[T, Any]:
-        """
-        Parses a chat completion using OpenAI's structured output format.
-        Returns both the parsed object and the raw completion for logprobs.
-        """
-        request_kwargs = {
-            "model": self._model,
-            "messages": message,
-            "response_format": output_model,
-            "temperature": temperature,
-        }
-
-        if logprobs:
-            request_kwargs["logprobs"] = True
-            request_kwargs["top_logprobs"] = top_logprobs
-
-        if priority:
-            request_kwargs["extra_body"] = {"priority": priority}
-
-        completion = self._client.beta.chat.completions.parse(**request_kwargs)
-        parsed = completion.choices[0].message.parsed
-        return parsed, completion
-
-    def run(
-        self,
-        # User parameters
-        text: str,
-        with_analysis: bool,
-        output_lang: str | None,
-        user_prompt: str | None,
-        temperature: float,
-        logprobs: bool,
-        top_logprobs: int | None,
-        validator: Callable[[Any], bool] | None,
-        max_validation_retries: int | None,
-        # Internal parameters
-        prompt_file: str,
-        output_model: Type[T],
-        mode: str | None,
-        priority: int | None = 0,
-        **extra_kwargs,
-    ) -> ToolOutput:
-        """
-        Execute the LLM pipeline with the given input text.
-        """
-        prompt_loader = PromptLoader()
-        formatter = Formatter()
-        output = ToolOutput()
-        try:
-            # Prompt configs contain two keys: main_template and analyze template, both are string
-            prompt_configs = prompt_loader.load(
-                prompt_file=prompt_file,
-                text=text.strip(),
-                mode=mode,
-                **extra_kwargs,
-            )
-
-            messages = []
-
-            if with_analysis:
-                analysis = self._analyze(prompt_configs, temperature)
-                messages.append(
-                    OperatorUtils.build_user_message(
-                        f"Based on this analysis: {analysis}"
-                    )
-                )
-
-            if output_lang:
-                messages.append(
-                    OperatorUtils.build_user_message(
-                        f"Respond only in the {output_lang} language."
-                    )
-                )
-
-            if user_prompt:
-                messages.append(
-                    OperatorUtils.build_user_message(
-                        f"Consider this instruction {user_prompt}"
-                    )
-                )
-
-            messages.append(
-                OperatorUtils.build_user_message(prompt_configs["main_template"])
-            )
-
-            messages = formatter.user_merge_format(messages)
-
-            parsed, completion = self._parse_completion(
-                messages, output_model, temperature, logprobs, top_logprobs, priority
-            )
-
-            output.result = parsed.result
-
-            # Retry logic if validation fails
-            if validator and not validator(output.result):
-                for attempt in range(max_validation_retries):
-                    logger.warning(
-                        f"Validation failed, retrying for the {attempt + 1} time."
-                    )
-
-                    # Generate new temperature for retry
-                    retry_temperature = OperatorUtils.get_retry_temp(temperature)
-                    try:
-                        parsed, completion = self._parse_completion(
-                            messages,
-                            output_model,
-                            retry_temperature,
-                            logprobs,
-                            top_logprobs,
-                        )
-
-                        output.result = parsed.result
-
-                        # Check if retry was successful
-                        if validator(output.result):
-                            logger.info(
-                                f"Validation passed on retry attempt {attempt + 1}"
-                            )
-                            break
-                        else:
-                            logger.warning(
-                                f"Validation still failing after retry attempt {attempt + 1}"
-                            )
-
-                    except Exception as e:
-                        logger.error(f"Retry attempt {attempt + 1} failed: {e}")
-                        # Continue to next retry attempt if this one fails
-
-            # Final check after all retries
-            if validator and not validator(output.result):
-                output.errors.append("Validation failed after all retry attempts")
-
-            if logprobs:
-                output.logprobs = OperatorUtils.extract_logprobs(completion)
-
-            if with_analysis:
-                output.analysis = analysis
-
-            output.process = prompt_file[:-5]
-
-            return output
-
-        except Exception as e:
-            logger.error(f"TheTool failed: {e}")
-            output.errors.append(str(e))
-            return output