hamtaa-texttools 2.1.0__py3-none-any.whl → 2.3.0__py3-none-any.whl

texttools/tools/sync_tools.py

@@ -5,17 +5,21 @@ from typing import Any, Literal
 
 from openai import OpenAI
 
-from ..core.exceptions import LLMError, PromptError, TextToolsError, ValidationError
-from ..core.internal_models import (
+from ..core import (
     Bool,
     ListDictStrStr,
     ListStr,
+    LLMError,
+    Operator,
+    PromptError,
     ReasonListStr,
     Str,
+    TextToolsError,
+    TheToolUtils,
+    TokenUsage,
+    ValidationError,
     create_dynamic_model,
 )
-from ..core.operators.sync_operator import Operator
-from ..core.utils import TheToolUtils
 from ..models import CategoryTree, ToolOutput, ToolOutputMetadata
 
 
@@ -29,6 +33,7 @@ class TheTool:
         self._operator = Operator(client=client, model=model)
         self.logger = logging.getLogger(self.__class__.__name__)
         self.raise_on_error = raise_on_error
+        self.model = model
 
     def categorize(
         self,
@@ -60,7 +65,6 @@ class TheTool:
 
         Returns:
             ToolOutput
-
         """
         tool_name = "categorize"
         start = perf_counter()
@@ -87,7 +91,10 @@ class TheTool:
                 )
 
                 metadata = ToolOutputMetadata(
-                    tool_name=tool_name, execution_time=perf_counter() - start
+                    tool_name=tool_name,
+                    execution_time=perf_counter() - start,
+                    processed_by=self.model,
+                    token_usage=operator_output.token_usage,
                 )
                 tool_output = ToolOutput(
                     result=operator_output.result,
@@ -102,6 +109,7 @@ class TheTool:
                 final_categories = []
                 analysis = ""
                 logprobs_list = []
+                token_usage = TokenUsage()
 
                 for _ in range(levels):
                     if not parent_node.children:
@@ -142,9 +150,13 @@ class TheTool:
                         analysis += level_operator_output.analysis
                     if logprobs:
                         logprobs_list.extend(level_operator_output.logprobs)
+                    token_usage += level_operator_output.token_usage
 
                 metadata = ToolOutputMetadata(
-                    tool_name=tool_name, execution_time=(perf_counter() - start)
+                    tool_name=tool_name,
+                    execution_time=perf_counter() - start,
+                    processed_by=self.model,
+                    token_usage=token_usage,
                 )
                 tool_output = ToolOutput(
                     result=final_categories,
@@ -225,7 +237,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -304,7 +319,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -378,7 +396,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -459,7 +480,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -538,7 +562,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -616,7 +643,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -692,7 +722,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -758,6 +791,7 @@ class TheTool:
                 translation = ""
                 analysis = ""
                 logprobs_list = []
+                token_usage = TokenUsage()
 
                 for chunk in chunks:
                     chunk_operator_output = self._operator.run(
@@ -785,9 +819,13 @@ class TheTool:
                         analysis += chunk_operator_output.analysis
                     if logprobs:
                         logprobs_list.extend(chunk_operator_output.logprobs)
+                    token_usage += chunk_operator_output.token_usage
 
                 metadata = ToolOutputMetadata(
-                    tool_name=tool_name, execution_time=perf_counter() - start
+                    tool_name=tool_name,
+                    execution_time=perf_counter() - start,
+                    processed_by=self.model,
+                    token_usage=token_usage,
                 )
                 tool_output = ToolOutput(
                     result=translation,
@@ -817,7 +855,10 @@ class TheTool:
                 )
 
                 metadata = ToolOutputMetadata(
-                    tool_name=tool_name, execution_time=perf_counter() - start
+                    tool_name=tool_name,
+                    execution_time=perf_counter() - start,
+                    processed_by=self.model,
+                    token_usage=operator_output.token_usage,
                 )
                 tool_output = ToolOutput(
                     result=operator_output.result,
@@ -895,7 +936,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -976,7 +1020,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,
@@ -1056,7 +1103,10 @@ class TheTool:
             )
 
             metadata = ToolOutputMetadata(
-                tool_name=tool_name, execution_time=perf_counter() - start
+                tool_name=tool_name,
+                execution_time=perf_counter() - start,
+                processed_by=self.model,
+                token_usage=operator_output.token_usage,
             )
             tool_output = ToolOutput(
                 result=operator_output.result,

hamtaa_texttools-2.1.0.dist-info/RECORD

@@ -1,30 +0,0 @@
-hamtaa_texttools-2.1.0.dist-info/licenses/LICENSE,sha256=gqxbR8wqI3utd__l3Yn6_dQ3Pou1a17W4KmydbvZGok,1084
-texttools/__init__.py,sha256=AHpTq1BbL3sWCaFiIjlSkqNfNqweq-qm2EIOSmUZRJ0,175
-texttools/models.py,sha256=CQnO1zkKHFyqeMWrYGA4IyXQ7YYLVc3Xz1WaXbXzDLw,4634
-texttools/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-texttools/core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-texttools/core/exceptions.py,sha256=6SDjUL1rmd3ngzD3ytF4LyTRj3bQMSFR9ECrLoqXXHw,395
-texttools/core/internal_models.py,sha256=CmRtXGZRn5fZ18lVb42N8LrZXvJb6WwdjIhgiotWJdA,1952
-texttools/core/utils.py,sha256=jqXHXU1DWDKWhK0HHSjnjq4_TLg3FMcnRzrwTF1eqqc,9744
-texttools/core/operators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-texttools/core/operators/async_operator.py,sha256=HOi9gUwIffJUtyp8WLNbMpxI8jnafNDrbtLl6vyPcUs,6221
-texttools/core/operators/sync_operator.py,sha256=yM14fsku-4Nf60lPUVePaB9Lu8HbGKb4ubwoizVWuYQ,6126
-texttools/prompts/augment.yaml,sha256=O-LMVyrihr0GQ8hp2Lx6uIR8Jh83bUDS9UZ-dvYOP7k,5453
-texttools/prompts/categorize.yaml,sha256=kN4uRPOC7q6A13bdCIox60vZZ8sgRiTtquv-kqIvTsk,1133
-texttools/prompts/extract_entities.yaml,sha256=-qe1eEvN-8nJ2_GLjeoFAPVORCPYUzsIt7UGXD485bE,648
-texttools/prompts/extract_keywords.yaml,sha256=jP74HFa4Dka01d1COStEBbdzW5onqwocwyyVsmNpECs,3276
-texttools/prompts/is_fact.yaml,sha256=kqF527DEdnlL3MG5tF1Z3ci_sRxmGv7dgNR2SuElq4Y,719
-texttools/prompts/is_question.yaml,sha256=C-ynlt0qHpUM4BAIh0oI7UJ5BxCNU9-GR9T5864jeto,496
-texttools/prompts/merge_questions.yaml,sha256=zgZs8BcwseZy1GsD_DvVGtw0yuCCc6xsK8VDmuHI2V0,1844
-texttools/prompts/propositionize.yaml,sha256=xTw3HQrxtxoMpkf8a9is0uZZ0AG4IDNfh7XE0aVlNso,1441
-texttools/prompts/run_custom.yaml,sha256=hSfR4BMJNUo9nP_AodPU7YTnhR-X_G-W7Pz0ROQzoI0,133
-texttools/prompts/summarize.yaml,sha256=0aKYFRDxODqOOEhSexi-hn3twLwkMFVmi7rtAifnCuA,464
-texttools/prompts/to_question.yaml,sha256=n8Bn28QjvSHwPHQLwRYpZ2IsaaBsq4pK9Dp_i0xk8eg,2210
-texttools/prompts/translate.yaml,sha256=omtC-TlFYMidy8WqRe7idUtKNiK4g3IhEl-iyufOwjk,649
-texttools/tools/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-texttools/tools/async_tools.py,sha256=2ZJ8K1-SSRSyyQ5VfDBZof0HDeRjEuakZJyHAlswrLw,46089
-texttools/tools/sync_tools.py,sha256=WqHaUQscOd6RbMCGjhFbC4muw1VZxu-W5qCOA9JIwVc,41835
-hamtaa_texttools-2.1.0.dist-info/METADATA,sha256=Sq4pywPSrBvHxp6sundpF2LFblcJqYgkhONx8V3XNyU,6958
-hamtaa_texttools-2.1.0.dist-info/WHEEL,sha256=wUyA8OaulRlbfwMtmQsvNngGrxQHAvkKcvRmdizlJi0,92
-hamtaa_texttools-2.1.0.dist-info/top_level.txt,sha256=5Mh0jIxxZ5rOXHGJ6Mp-JPKviywwN0MYuH0xk5bEWqE,10
-hamtaa_texttools-2.1.0.dist-info/RECORD,,