hamtaa-texttools 1.1.19__py3-none-any.whl → 1.1.21__py3-none-any.whl

texttools/internals/models.py

@@ -1,44 +1,58 @@
+from __future__ import annotations
+
 from datetime import datetime
-from typing import Type, Any, Literal
+from typing import Type, Literal, Any
 
 from pydantic import BaseModel, Field, create_model
 
 
+class ToolOutputMetadata(BaseModel):
+    tool_name: str
+    processed_at: datetime = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+    execution_time: float | None = None
+
+
 class ToolOutput(BaseModel):
     result: Any = None
-    logprobs: list[dict[str, Any]] = []
-    analysis: str = ""
-    process: str | None = None
-    processed_at: datetime = datetime.now()
-    execution_time: float | None = None
+    analysis: str | None = None
+    logprobs: list[dict[str, Any]] | None = None
     errors: list[str] = []
+    metadata: ToolOutputMetadata | None = None
 
     def __repr__(self) -> str:
-        return f"""
-        ToolOutput(process='{self.process}', result_type='{type(self.result)}', 
-        result='{self.result}', analysis='{self.analysis}', 
-        logprobs='{self.logprobs}', errors='{self.errors}', 
-        processed_at='{self.processed_at}', execution_time='{self.execution_time}'
-        """
+        base = f"""ToolOutput(result='{self.result}', result_type='{type(self.result)}', analysis='{self.analysis}', logprobs='{self.logprobs}', errors='{self.errors}'"""
 
+        if self.metadata:
+            base += f""", tool_name='{self.metadata.tool_name}', 
+            processed_at='{self.metadata.processed_at}', execution_time='{self.metadata.execution_time}'
+            """
 
-class StrOutput(BaseModel):
+        return base
+
+
+class OperatorOutput(BaseModel):
+    result: Any
+    analysis: str | None
+    logprobs: list[dict[str, Any]] | None
+
+
+class Str(BaseModel):
     result: str = Field(..., description="The output string", example="text")
 
 
-class BoolOutput(BaseModel):
+class Bool(BaseModel):
     result: bool = Field(
         ..., description="Boolean indicating the output state", example=True
     )
 
 
-class ListStrOutput(BaseModel):
+class ListStr(BaseModel):
     result: list[str] = Field(
         ..., description="The output list of strings", example=["text_1", "text_2"]
     )
 
 
-class ListDictStrStrOutput(BaseModel):
+class ListDictStrStr(BaseModel):
     result: list[dict[str, str]] = Field(
         ...,
         description="List of dictionaries containing string key-value pairs",
@@ -46,121 +60,76 @@ class ListDictStrStrOutput(BaseModel):
     )
 
 
-class ReasonListStrOutput(BaseModel):
+class ReasonListStr(BaseModel):
     reason: str = Field(..., description="Thinking process that led to the output")
     result: list[str] = Field(
         ..., description="The output list of strings", example=["text_1", "text_2"]
     )
 
 
-class Node(BaseModel):
-    node_id: int
-    name: str
-    level: int
-    parent_id: int | None
-    description: str
+class Node:
+    def __init__(self, name: str, description: str, level: int, parent: Node | None):
+        self.name = name
+        self.description = description
+        self.level = level
+        self.parent = parent
+        self.children = {}
 
 
 class CategoryTree:
-    def __init__(self, tree_name):
-        self._root = Node(
-            node_id=0, name=tree_name, level=0, parent_id=None, description="Root node"
-        )
-        self._all_nodes: list[Node] = [self._root]
-        self._new_id = 1
-
-    def get_all_nodes(self) -> list[Node]:
+    def __init__(self):
+        self._root = Node(name="root", description="root", level=0, parent=None)
+        self._all_nodes = {"root": self._root}
+
+    def get_all_nodes(self) -> dict[str, Node]:
         return self._all_nodes
 
     def get_level_count(self) -> int:
-        return max([item.level for item in self._all_nodes])
-
-    def get_node(self, identifier: int | str) -> Node | None:
-        if isinstance(identifier, str):
-            for node in self.get_all_nodes():
-                if node.name == identifier:
-                    return node
-            return None
-        elif isinstance(identifier, int):
-            for node in self.get_all_nodes():
-                if node.node_id == identifier:
-                    return node
-            return None
-        else:
-            return None
-
-    def get_children(self, parent_node: Node) -> list[Node] | None:
-        children = [
-            node
-            for node in self.get_all_nodes()
-            if parent_node.node_id == node.parent_id
-        ]
-        return children if children else None
+        return max(node.level for node in self._all_nodes.values())
+
+    def get_node(self, name: str) -> Node | None:
+        return self._all_nodes.get(name)
 
     def add_node(
         self,
-        node_name: str,
-        parent_name: str | None = None,
+        name: str,
+        parent_name: str,
         description: str | None = None,
     ) -> None:
-        if self.get_node(node_name):
-            raise ValueError(f"{node_name} has been chosen for another category before")
-
-        if parent_name:
-            parent_node = self.get_node(parent_name)
-            if not parent_node:
-                raise ValueError(f"Parent category '{parent_name}' not found")
-            parent_id = parent_node.node_id
-            level = parent_node.level + 1
-        else:
-            level = 1
-            parent_id = 0
+        if self.get_node(name):
+            raise ValueError(f"Cannot add {name} category twice")
+
+        parent = self.get_node(parent_name)
+
+        if not parent:
+            raise ValueError(f"Parent category '{parent_name}' not found")
 
         node_data = {
-            "node_id": self._new_id,
-            "name": node_name,
-            "level": level,
-            "parent_id": parent_id,
+            "name": name,
             "description": description if description else "No description provided",
+            "level": parent.level + 1,
+            "parent": parent,
         }
 
-        self._all_nodes.append(Node(**node_data))
-        self._new_id += 1
-
-    def remove_node(self, identifier: int | str) -> None:
-        node = self.get_node(identifier)
-
-        if node:
-            # Remove node's children recursively
-            children = self.get_children(node)
+        new_node = Node(**node_data)
+        parent.children[name] = new_node
+        self._all_nodes[name] = new_node
 
-            if not children:
-                self._all_nodes.remove(node)
-                return
+    def remove_node(self, name: str) -> None:
+        if name == "root":
+            raise ValueError("Cannot remove the root node")
 
-            for child in children:
-                self.remove_node(child.name)
+        node = self.get_node(name)
+        if not node:
+            raise ValueError(f"Category: '{name}' not found")
 
-            self._all_nodes.remove(node)
-        else:
-            raise ValueError(f"Node with identifier: '{identifier}' not found.")
+        for child_name in list(node.children.keys()):
+            self.remove_node(child_name)
 
-    def dump_tree(self) -> dict:
-        def build_dict(node: Node) -> dict:
-            children = [
-                build_dict(child)
-                for child in self._all_nodes
-                if child.parent_id == node.node_id
-            ]
-            return {
-                "node_id": node.node_id,
-                "name": node.name,
-                "level": node.level,
-                "parent_id": node.parent_id,
-                "children": children,
-            }
+        if node.parent:
+            del node.parent.children[name]
 
-        return {"category_tree": build_dict(self._root)["children"]}
+        del self._all_nodes[name]
 
 
 # This function is needed to create CategorizerOutput with dynamic categories
@@ -179,12 +148,3 @@ def create_dynamic_model(allowed_values: list[str]) -> Type[BaseModel]:
     )
 
     return CategorizerOutput
-
-
-class Entity(BaseModel):
-    text: str = Field(description="The exact text of the entity")
-    entity_type: str = Field(description="The type of the entity")
-
-
-class EntityDetectorOutput(BaseModel):
-    result: list[Entity] = Field(description="List of all extracted entities")

texttools/internals/operator_utils.py

@@ -5,8 +5,8 @@ import random
 
 class OperatorUtils:
     @staticmethod
-    def build_user_message(prompt: str) -> dict[str, str]:
-        return {"role": "user", "content": prompt}
+    def build_user_message(prompt: str) -> list[dict[str, str]]:
+        return [{"role": "user", "content": prompt}]
 
     @staticmethod
     def extract_logprobs(completion: dict) -> list[dict]:

texttools/internals/prompt_loader.py

@@ -12,20 +12,12 @@ class PromptLoader:
     Responsibilities:
     - Load and parse YAML prompt definitions.
     - Select the right template (by mode, if applicable).
-    - Inject variables (`{input}`, plus any extra kwargs) into the templates.
+    - Inject variables (`{text}`, plus any extra kwargs) into the templates.
     """
 
     MAIN_TEMPLATE = "main_template"
     ANALYZE_TEMPLATE = "analyze_template"
 
-    @staticmethod
-    def _build_format_args(text: str, **extra_kwargs) -> dict[str, str]:
-        # Base formatting args
-        format_args = {"input": text}
-        # Merge extras
-        format_args.update(extra_kwargs)
-        return format_args
-
     # Use lru_cache to load each file once
     @lru_cache(maxsize=32)
     def _load_templates(self, prompt_file: str, mode: str | None) -> dict[str, str]:
@@ -69,16 +61,6 @@ class PromptLoader:
                     + (f" for mode '{mode}'" if mode else "")
                 )
 
-            if (
-                not analyze_template
-                or not analyze_template.strip()
-                or analyze_template.strip() in ["{analyze_template}", "{}"]
-            ):
-                raise PromptError(
-                    "analyze_template cannot be empty"
-                    + (f" for mode '{mode}'" if mode else "")
-                )
-
             return {
                 self.MAIN_TEMPLATE: main_template,
                 self.ANALYZE_TEMPLATE: analyze_template,
@@ -94,7 +76,8 @@ class PromptLoader:
     ) -> dict[str, str]:
         try:
             template_configs = self._load_templates(prompt_file, mode)
-            format_args = self._build_format_args(text, **extra_kwargs)
+            format_args = {"text": text}
+            format_args.update(extra_kwargs)
 
             # Inject variables inside each template
             for key in template_configs.keys():

texttools/internals/sync_operator.py

@@ -1,13 +1,11 @@
-from typing import Any, TypeVar, Type
+from typing import TypeVar, Type
 from collections.abc import Callable
-import logging
 
 from openai import OpenAI
 from pydantic import BaseModel
 
-from texttools.internals.models import ToolOutput
+from texttools.internals.models import OperatorOutput
 from texttools.internals.operator_utils import OperatorUtils
-from texttools.internals.formatters import Formatter
 from texttools.internals.prompt_loader import PromptLoader
 from texttools.internals.exceptions import (
     TextToolsError,
@@ -19,35 +17,23 @@ from texttools.internals.exceptions import (
 # Base Model type for output models
 T = TypeVar("T", bound=BaseModel)
 
-logger = logging.getLogger("texttools.sync_operator")
-
 
 class Operator:
     """
-    Core engine for running text-processing operations with an LLM (Sync).
-
-    It wires together:
-    - `PromptLoader` → loads YAML prompt templates.
-    - `UserMergeFormatter` → applies formatting to messages (e.g., merging).
-    - OpenAI client → executes completions/parsed completions.
+    Core engine for running text-processing operations with an LLM.
     """
 
     def __init__(self, client: OpenAI, model: str):
         self._client = client
         self._model = model
 
-    def _analyze(self, prompt_configs: dict[str, str], temperature: float) -> str:
-        """
-        Calls OpenAI API for analysis using the configured prompt template.
-        Returns the analyzed content as a string.
-        """
+    def _analyze_completion(self, analyze_prompt: str, temperature: float) -> str:
         try:
-            analyze_prompt = prompt_configs["analyze_template"]
-
             if not analyze_prompt:
                 raise PromptError("Analyze template is empty")
 
-            analyze_message = [OperatorUtils.build_user_message(analyze_prompt)]
+            analyze_message = OperatorUtils.build_user_message(analyze_prompt)
+
             completion = self._client.chat.completions.create(
                 model=self._model,
                 messages=analyze_message,
@@ -62,7 +48,7 @@ class Operator:
             if not analysis:
                 raise LLMError("Empty analysis response")
 
-            return analysis.strip()
+            return analysis
 
         except Exception as e:
             if isinstance(e, (PromptError, LLMError)):
@@ -71,21 +57,23 @@ class Operator:
 
     def _parse_completion(
         self,
-        message: list[dict[str, str]],
+        main_prompt: str,
         output_model: Type[T],
         temperature: float,
-        logprobs: bool = False,
-        top_logprobs: int = 3,
-        priority: int | None = 0,
-    ) -> tuple[T, Any]:
+        logprobs: bool,
+        top_logprobs: int,
+        priority: int,
+    ) -> tuple[T, object]:
         """
         Parses a chat completion using OpenAI's structured output format.
         Returns both the parsed object and the raw completion for logprobs.
         """
         try:
+            main_message = OperatorUtils.build_user_message(main_prompt)
+
             request_kwargs = {
                 "model": self._model,
-                "messages": message,
+                "messages": main_message,
                 "response_format": output_model,
                 "temperature": temperature,
             }
@@ -93,8 +81,10 @@ class Operator:
             if logprobs:
                 request_kwargs["logprobs"] = True
                 request_kwargs["top_logprobs"] = top_logprobs
+
             if priority:
                 request_kwargs["extra_body"] = {"priority": priority}
+
             completion = self._client.beta.chat.completions.parse(**request_kwargs)
 
             if not completion.choices:
@@ -121,25 +111,22 @@ class Operator:
         user_prompt: str | None,
         temperature: float,
         logprobs: bool,
-        top_logprobs: int | None,
-        validator: Callable[[Any], bool] | None,
+        top_logprobs: int,
+        validator: Callable[[object], bool] | None,
         max_validation_retries: int | None,
+        priority: int,
         # Internal parameters
         prompt_file: str,
         output_model: Type[T],
         mode: str | None,
-        priority: int | None = 0,
         **extra_kwargs,
-    ) -> ToolOutput:
+    ) -> OperatorOutput:
         """
         Execute the LLM pipeline with the given input text. (Sync)
         """
         try:
             prompt_loader = PromptLoader()
-            formatter = Formatter()
-            output = ToolOutput()
 
-            # Prompt configs contain two keys: main_template and analyze template, both are string
             prompt_configs = prompt_loader.load(
                 prompt_file=prompt_file,
                 text=text.strip(),
@@ -147,47 +134,32 @@ class Operator:
                 **extra_kwargs,
             )
 
-            messages = []
+            main_prompt = ""
+            analysis = ""
 
             if with_analysis:
-                analysis = self._analyze(prompt_configs, temperature)
-                messages.append(
-                    OperatorUtils.build_user_message(
-                        f"Based on this analysis: {analysis}"
-                    )
+                analysis = self._analyze_completion(
+                    prompt_configs["analyze_template"], temperature
                 )
+                main_prompt += f"Based on this analysis:\n{analysis}\n"
 
             if output_lang:
-                messages.append(
-                    OperatorUtils.build_user_message(
-                        f"Respond only in the {output_lang} language."
-                    )
-                )
+                main_prompt += f"Respond only in the {output_lang} language.\n"
 
             if user_prompt:
-                messages.append(
-                    OperatorUtils.build_user_message(
-                        f"Consider this instruction {user_prompt}"
-                    )
-                )
-
-            messages.append(
-                OperatorUtils.build_user_message(prompt_configs["main_template"])
-            )
+                main_prompt += f"Consider this instruction {user_prompt}\n"
 
-            messages = formatter.user_merge_format(messages)
+            main_prompt += prompt_configs["main_template"]
 
             if logprobs and (not isinstance(top_logprobs, int) or top_logprobs < 2):
                 raise ValueError("top_logprobs should be an integer greater than 1")
 
             parsed, completion = self._parse_completion(
-                messages, output_model, temperature, logprobs, top_logprobs, priority
+                main_prompt, output_model, temperature, logprobs, top_logprobs, priority
             )
 
-            output.result = parsed.result
-
             # Retry logic if validation fails
-            if validator and not validator(output.result):
+            if validator and not validator(parsed.result):
                 if (
                     not isinstance(max_validation_retries, int)
                     or max_validation_retries < 1
@@ -197,17 +169,13 @@ class Operator:
                     )
 
                 succeeded = False
-                for attempt in range(max_validation_retries):
-                    logger.warning(
-                        f"Validation failed, retrying for the {attempt + 1} time."
-                    )
-
-                    # Generate new temperature for retry
+                for _ in range(max_validation_retries):
+                    # Generate a new temperature to retry
                     retry_temperature = OperatorUtils.get_retry_temp(temperature)
 
                     try:
                         parsed, completion = self._parse_completion(
-                            messages,
+                            main_prompt,
                             output_model,
                             retry_temperature,
                             logprobs,
@@ -215,30 +183,26 @@ class Operator:
                             priority=priority,
                         )
 
-                        output.result = parsed.result
-
                         # Check if retry was successful
-                        if validator(output.result):
+                        if validator(parsed.result):
                             succeeded = True
                             break
 
-                    except LLMError as e:
-                        logger.error(f"Retry attempt {attempt + 1} failed: {e}")
+                    except LLMError:
+                        pass
 
                 if not succeeded:
-                    raise ValidationError(
-                        f"Validation failed after {max_validation_retries} retries"
-                    )
-
-            if logprobs:
-                output.logprobs = OperatorUtils.extract_logprobs(completion)
-
-            if with_analysis:
-                output.analysis = analysis
-
-            output.process = prompt_file[:-5]
+                    raise ValidationError("Validation failed after all retries")
+
+            operator_output = OperatorOutput(
+                result=parsed.result,
+                analysis=analysis if with_analysis else None,
+                logprobs=OperatorUtils.extract_logprobs(completion)
+                if logprobs
+                else None,
+            )
 
-            return output
+            return operator_output
 
         except (PromptError, LLMError, ValidationError):
             raise

texttools/internals/text_to_chunks.py

@@ -0,0 +1,97 @@
+import re
+
+
+def text_to_chunks(text: str, size: int, overlap: int) -> list[str]:
+    separators = ["\n\n", "\n", " ", ""]
+    is_separator_regex = False
+    keep_separator = True  # Equivalent to 'start'
+    length_function = len
+    strip_whitespace = True
+    chunk_size = size
+    chunk_overlap = overlap
+
+    def _split_text_with_regex(
+        text: str, separator: str, keep_separator: bool
+    ) -> list[str]:
+        if not separator:
+            return [text]
+        if not keep_separator:
+            return re.split(separator, text)
+        _splits = re.split(f"({separator})", text)
+        splits = [_splits[i] + _splits[i + 1] for i in range(1, len(_splits), 2)]
+        if len(_splits) % 2 == 0:
+            splits += [_splits[-1]]
+        return [_splits[0]] + splits if _splits[0] else splits
+
+    def _join_docs(docs: list[str], separator: str) -> str | None:
+        text = separator.join(docs)
+        if strip_whitespace:
+            text = text.strip()
+        return text if text else None
+
+    def _merge_splits(splits: list[str], separator: str) -> list[str]:
+        separator_len = length_function(separator)
+        docs = []
+        current_doc = []
+        total = 0
+        for d in splits:
+            len_ = length_function(d)
+            if total + len_ + (separator_len if current_doc else 0) > chunk_size:
+                if total > chunk_size:
+                    pass
+                if current_doc:
+                    doc = _join_docs(current_doc, separator)
+                    if doc is not None:
+                        docs.append(doc)
+                    while total > chunk_overlap or (
+                        total + len_ + (separator_len if current_doc else 0)
+                        > chunk_size
+                        and total > 0
+                    ):
+                        total -= length_function(current_doc[0]) + (
+                            separator_len if len(current_doc) > 1 else 0
+                        )
+                        current_doc = current_doc[1:]
+            current_doc.append(d)
+            total += len_ + (separator_len if len(current_doc) > 1 else 0)
+        doc = _join_docs(current_doc, separator)
+        if doc is not None:
+            docs.append(doc)
+        return docs
+
+    def _split_text(text: str, separators: list[str]) -> list[str]:
+        final_chunks = []
+        separator = separators[-1]
+        new_separators = []
+        for i, _s in enumerate(separators):
+            separator_ = _s if is_separator_regex else re.escape(_s)
+            if not _s:
+                separator = _s
+                break
+            if re.search(separator_, text):
+                separator = _s
+                new_separators = separators[i + 1 :]
+                break
+        separator_ = separator if is_separator_regex else re.escape(separator)
+        splits = _split_text_with_regex(text, separator_, keep_separator)
+        _separator = "" if keep_separator else separator
+        good_splits = []
+        for s in splits:
+            if length_function(s) < chunk_size:
+                good_splits.append(s)
+            else:
+                if good_splits:
+                    merged_text = _merge_splits(good_splits, _separator)
+                    final_chunks.extend(merged_text)
+                    good_splits = []
+                if not new_separators:
+                    final_chunks.append(s)
+                else:
+                    other_info = _split_text(s, new_separators)
+                    final_chunks.extend(other_info)
+        if good_splits:
+            merged_text = _merge_splits(good_splits, _separator)
+            final_chunks.extend(merged_text)
+        return final_chunks
+
+    return _split_text(text, separators)

texttools/prompts/README.md

@@ -15,7 +15,7 @@ This folder contains YAML files for all prompts used in the project. Each file r
 ```yaml
 main_template:
   mode_1: |
-    Your main instructions here with placeholders like {input}.
+    Your main instructions here with placeholders like {text}.
   mode_2: |
     Optional reasoning instructions here.
 
@@ -30,6 +30,6 @@ analyze_template:
 
 ## Guidelines
 1. **Naming**: Use descriptive names for each YAML file corresponding to the tool or task it serves.
-2. **Placeholders**: Use `{input}` or other relevant placeholders to dynamically inject data.
+2. **Placeholders**: Use `{text}` or other relevant placeholders to dynamically inject data.
 3. **Modes**: If using modes, ensure both `main_template` and `analyze_template` contain the corresponding keys.
 4. **Consistency**: Keep formatting consistent across files for easier parsing by scripts.