hamtaa-texttools 1.0.8__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of hamtaa-texttools might be problematic. Click here for more details.
- {hamtaa_texttools-1.0.8/hamtaa_texttools.egg-info → hamtaa_texttools-1.1.0}/PKG-INFO +9 -9
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/README.md +7 -7
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0/hamtaa_texttools.egg-info}/PKG-INFO +9 -9
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/pyproject.toml +2 -2
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/async_operator.py +10 -8
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/base_operator.py +7 -1
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/operator.py +8 -6
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/output_models.py +8 -1
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/LICENSE +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/MANIFEST.in +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/hamtaa_texttools.egg-info/SOURCES.txt +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/hamtaa_texttools.egg-info/dependency_links.txt +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/hamtaa_texttools.egg-info/requires.txt +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/hamtaa_texttools.egg-info/top_level.txt +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/setup.cfg +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/__init__.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/batch/__init__.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/batch/batch_manager.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/batch/batch_runner.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/README.md +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/categorizer.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/extract_entities.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/extract_keywords.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/is_question.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/merge_questions.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/rewrite.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/run_custom.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/subject_to_question.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/summarize.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/text_to_question.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/translate.yaml +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/__init__.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/async_the_tool.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/formatters.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/prompt_loader.py +0 -0
- {hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/the_tool.py +0 -0
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hamtaa-texttools
|
|
3
|
-
Version: 1.0
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: A high-level NLP toolkit built on top of modern LLMs.
|
|
5
5
|
Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, MoosaviNejad <erfanmoosavi84@gmail.com>
|
|
6
6
|
License: MIT License
|
|
7
7
|
|
|
@@ -63,7 +63,7 @@ Each tool is designed to work out-of-the-box with structured outputs (JSON / Pyd
|
|
|
63
63
|
|
|
64
64
|
---
|
|
65
65
|
|
|
66
|
-
## ⚙️ `with_analysis`, `logprobs`, `output_lang`, and `
|
|
66
|
+
## ⚙️ `with_analysis`, `logprobs`, `output_lang`, `user_prompt` and `temperature` parameters
|
|
67
67
|
|
|
68
68
|
TextTools provides several optional flags to customize LLM behavior:
|
|
69
69
|
|
|
@@ -119,14 +119,14 @@ the_tool = TheTool(client=client, model=model)
|
|
|
119
119
|
|
|
120
120
|
# Example: Question Detection
|
|
121
121
|
detection = the_tool.is_question("Is this project open source?", logprobs=True, top_logprobs=2)
|
|
122
|
-
print(detection
|
|
123
|
-
print(detection
|
|
122
|
+
print(detection.result)
|
|
123
|
+
print(detection.logprobs)
|
|
124
124
|
# Output: True \n --logprobs
|
|
125
125
|
|
|
126
126
|
# Example: Translation
|
|
127
127
|
translation = the_tool.translate("سلام، حالت چطوره؟" target_language="English", with_analysis=True)
|
|
128
|
-
print(translation
|
|
129
|
-
print(translation
|
|
128
|
+
print(translation.result)
|
|
129
|
+
print(translation.analysis)
|
|
130
130
|
# Output: "Hi! How are you?" \n --analysis
|
|
131
131
|
```
|
|
132
132
|
|
|
@@ -150,8 +150,8 @@ async def main():
|
|
|
150
150
|
the_tool = AsyncTheTool(client=async_client, model=model)
|
|
151
151
|
|
|
152
152
|
# Example: Async Translation
|
|
153
|
-
|
|
154
|
-
print(result
|
|
153
|
+
translation = await the_tool.translate("سلام، حالت چطوره؟", target_language="English")
|
|
154
|
+
print(translation.result)
|
|
155
155
|
# Output: "Hi! How are you?"
|
|
156
156
|
|
|
157
157
|
asyncio.run(main())
|
|
@@ -29,7 +29,7 @@ Each tool is designed to work out-of-the-box with structured outputs (JSON / Pyd
|
|
|
29
29
|
|
|
30
30
|
---
|
|
31
31
|
|
|
32
|
-
## ⚙️ `with_analysis`, `logprobs`, `output_lang`, and `
|
|
32
|
+
## ⚙️ `with_analysis`, `logprobs`, `output_lang`, `user_prompt` and `temperature` parameters
|
|
33
33
|
|
|
34
34
|
TextTools provides several optional flags to customize LLM behavior:
|
|
35
35
|
|
|
@@ -85,14 +85,14 @@ the_tool = TheTool(client=client, model=model)
|
|
|
85
85
|
|
|
86
86
|
# Example: Question Detection
|
|
87
87
|
detection = the_tool.is_question("Is this project open source?", logprobs=True, top_logprobs=2)
|
|
88
|
-
print(detection
|
|
89
|
-
print(detection
|
|
88
|
+
print(detection.result)
|
|
89
|
+
print(detection.logprobs)
|
|
90
90
|
# Output: True \n --logprobs
|
|
91
91
|
|
|
92
92
|
# Example: Translation
|
|
93
93
|
translation = the_tool.translate("سلام، حالت چطوره؟" target_language="English", with_analysis=True)
|
|
94
|
-
print(translation
|
|
95
|
-
print(translation
|
|
94
|
+
print(translation.result)
|
|
95
|
+
print(translation.analysis)
|
|
96
96
|
# Output: "Hi! How are you?" \n --analysis
|
|
97
97
|
```
|
|
98
98
|
|
|
@@ -116,8 +116,8 @@ async def main():
|
|
|
116
116
|
the_tool = AsyncTheTool(client=async_client, model=model)
|
|
117
117
|
|
|
118
118
|
# Example: Async Translation
|
|
119
|
-
|
|
120
|
-
print(result
|
|
119
|
+
translation = await the_tool.translate("سلام، حالت چطوره؟", target_language="English")
|
|
120
|
+
print(translation.result)
|
|
121
121
|
# Output: "Hi! How are you?"
|
|
122
122
|
|
|
123
123
|
asyncio.run(main())
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hamtaa-texttools
|
|
3
|
-
Version: 1.0
|
|
4
|
-
Summary:
|
|
3
|
+
Version: 1.1.0
|
|
4
|
+
Summary: A high-level NLP toolkit built on top of modern LLMs.
|
|
5
5
|
Author-email: Tohidi <the.mohammad.tohidi@gmail.com>, Montazer <montazerh82@gmail.com>, Givechi <mohamad.m.givechi@gmail.com>, MoosaviNejad <erfanmoosavi84@gmail.com>
|
|
6
6
|
License: MIT License
|
|
7
7
|
|
|
@@ -63,7 +63,7 @@ Each tool is designed to work out-of-the-box with structured outputs (JSON / Pyd
|
|
|
63
63
|
|
|
64
64
|
---
|
|
65
65
|
|
|
66
|
-
## ⚙️ `with_analysis`, `logprobs`, `output_lang`, and `
|
|
66
|
+
## ⚙️ `with_analysis`, `logprobs`, `output_lang`, `user_prompt` and `temperature` parameters
|
|
67
67
|
|
|
68
68
|
TextTools provides several optional flags to customize LLM behavior:
|
|
69
69
|
|
|
@@ -119,14 +119,14 @@ the_tool = TheTool(client=client, model=model)
|
|
|
119
119
|
|
|
120
120
|
# Example: Question Detection
|
|
121
121
|
detection = the_tool.is_question("Is this project open source?", logprobs=True, top_logprobs=2)
|
|
122
|
-
print(detection
|
|
123
|
-
print(detection
|
|
122
|
+
print(detection.result)
|
|
123
|
+
print(detection.logprobs)
|
|
124
124
|
# Output: True \n --logprobs
|
|
125
125
|
|
|
126
126
|
# Example: Translation
|
|
127
127
|
translation = the_tool.translate("سلام، حالت چطوره؟" target_language="English", with_analysis=True)
|
|
128
|
-
print(translation
|
|
129
|
-
print(translation
|
|
128
|
+
print(translation.result)
|
|
129
|
+
print(translation.analysis)
|
|
130
130
|
# Output: "Hi! How are you?" \n --analysis
|
|
131
131
|
```
|
|
132
132
|
|
|
@@ -150,8 +150,8 @@ async def main():
|
|
|
150
150
|
the_tool = AsyncTheTool(client=async_client, model=model)
|
|
151
151
|
|
|
152
152
|
# Example: Async Translation
|
|
153
|
-
|
|
154
|
-
print(result
|
|
153
|
+
translation = await the_tool.translate("سلام، حالت چطوره؟", target_language="English")
|
|
154
|
+
print(translation.result)
|
|
155
155
|
# Output: "Hi! How are you?"
|
|
156
156
|
|
|
157
157
|
asyncio.run(main())
|
|
@@ -4,14 +4,14 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "hamtaa-texttools"
|
|
7
|
-
version = "1.0
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
authors = [
|
|
9
9
|
{ name = "Tohidi", email = "the.mohammad.tohidi@gmail.com" },
|
|
10
10
|
{ name = "Montazer", email = "montazerh82@gmail.com" },
|
|
11
11
|
{ name = "Givechi", email = "mohamad.m.givechi@gmail.com" },
|
|
12
12
|
{ name = "MoosaviNejad", email = "erfanmoosavi84@gmail.com" },
|
|
13
13
|
]
|
|
14
|
-
description = "
|
|
14
|
+
description = "A high-level NLP toolkit built on top of modern LLMs."
|
|
15
15
|
readme = "README.md"
|
|
16
16
|
license = {file = "LICENSE"}
|
|
17
17
|
requires-python = ">=3.8"
|
{hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/async_operator.py
RENAMED
|
@@ -4,6 +4,7 @@ import logging
|
|
|
4
4
|
from openai import AsyncOpenAI
|
|
5
5
|
from pydantic import BaseModel
|
|
6
6
|
|
|
7
|
+
from texttools.tools.internals.output_models import ToolOutput
|
|
7
8
|
from texttools.tools.internals.base_operator import BaseOperator
|
|
8
9
|
from texttools.tools.internals.formatters import Formatter
|
|
9
10
|
from texttools.tools.internals.prompt_loader import PromptLoader
|
|
@@ -152,7 +153,7 @@ class AsyncOperator(BaseOperator):
|
|
|
152
153
|
messages, output_model, temperature, logprobs, top_logprobs
|
|
153
154
|
)
|
|
154
155
|
elif resp_format == "parse":
|
|
155
|
-
parsed, completion = await self.
|
|
156
|
+
parsed, completion = await self._parse_completion(
|
|
156
157
|
messages, output_model, temperature, logprobs, top_logprobs
|
|
157
158
|
)
|
|
158
159
|
|
|
@@ -162,16 +163,17 @@ class AsyncOperator(BaseOperator):
|
|
|
162
163
|
"The provided output_model must define a field named 'result'"
|
|
163
164
|
)
|
|
164
165
|
|
|
165
|
-
|
|
166
|
+
output = ToolOutput(result="", analysis="", logprobs=[], errors=[])
|
|
167
|
+
|
|
168
|
+
output.result = parsed.result
|
|
166
169
|
|
|
167
170
|
if logprobs:
|
|
168
|
-
|
|
171
|
+
output.logprobs = self._extract_logprobs(completion)
|
|
169
172
|
|
|
170
173
|
if with_analysis:
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
return results
|
|
174
|
+
output.analysis = analysis
|
|
174
175
|
|
|
176
|
+
return output
|
|
175
177
|
except Exception as e:
|
|
176
|
-
logger.error(f"
|
|
177
|
-
return
|
|
178
|
+
logger.error(f"AsyncTheTool failed: {e}")
|
|
179
|
+
return ToolOutput(result="", analysis="", logprobs=[], errors=[str(e)])
|
{hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/base_operator.py
RENAMED
|
@@ -2,6 +2,7 @@ from typing import TypeVar, Type, Any
|
|
|
2
2
|
import json
|
|
3
3
|
import re
|
|
4
4
|
import math
|
|
5
|
+
import logging
|
|
5
6
|
|
|
6
7
|
from pydantic import BaseModel
|
|
7
8
|
from openai import OpenAI, AsyncOpenAI
|
|
@@ -9,6 +10,10 @@ from openai import OpenAI, AsyncOpenAI
|
|
|
9
10
|
# Base Model type for output models
|
|
10
11
|
T = TypeVar("T", bound=BaseModel)
|
|
11
12
|
|
|
13
|
+
# Configure logger
|
|
14
|
+
logger = logging.getLogger("base_operator")
|
|
15
|
+
logger.setLevel(logging.INFO)
|
|
16
|
+
|
|
12
17
|
|
|
13
18
|
class BaseOperator:
|
|
14
19
|
def __init__(self, client: OpenAI | AsyncOpenAI, model: str):
|
|
@@ -61,7 +66,8 @@ class BaseOperator:
|
|
|
61
66
|
|
|
62
67
|
for choice in completion.choices:
|
|
63
68
|
if not getattr(choice, "logprobs", None):
|
|
64
|
-
|
|
69
|
+
logger.error("logprobs is not avalible in the chosen model.")
|
|
70
|
+
return []
|
|
65
71
|
|
|
66
72
|
for logprob_item in choice.logprobs.content:
|
|
67
73
|
if ignore_pattern.match(logprob_item.token):
|
|
@@ -4,6 +4,7 @@ import logging
|
|
|
4
4
|
from openai import OpenAI
|
|
5
5
|
from pydantic import BaseModel
|
|
6
6
|
|
|
7
|
+
from texttools.tools.internals.output_models import ToolOutput
|
|
7
8
|
from texttools.tools.internals.base_operator import BaseOperator
|
|
8
9
|
from texttools.tools.internals.formatters import Formatter
|
|
9
10
|
from texttools.tools.internals.prompt_loader import PromptLoader
|
|
@@ -162,16 +163,17 @@ class Operator(BaseOperator):
|
|
|
162
163
|
"The provided output_model must define a field named 'result'"
|
|
163
164
|
)
|
|
164
165
|
|
|
165
|
-
|
|
166
|
+
output = ToolOutput(result="", analysis="", logprobs=[], errors=[])
|
|
167
|
+
|
|
168
|
+
output.result = parsed.result
|
|
166
169
|
|
|
167
170
|
if logprobs:
|
|
168
|
-
|
|
171
|
+
output.logprobs = self._extract_logprobs(completion)
|
|
169
172
|
|
|
170
173
|
if with_analysis:
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
return result
|
|
174
|
+
output.analysis = analysis
|
|
174
175
|
|
|
176
|
+
return output
|
|
175
177
|
except Exception as e:
|
|
176
178
|
logger.error(f"TheTool failed: {e}")
|
|
177
|
-
return
|
|
179
|
+
return ToolOutput(result="", analysis="", logprobs=[], errors=[str(e)])
|
{hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/output_models.py
RENAMED
|
@@ -1,8 +1,15 @@
|
|
|
1
|
-
from typing import Literal
|
|
1
|
+
from typing import Literal, Any
|
|
2
2
|
|
|
3
3
|
from pydantic import BaseModel, Field
|
|
4
4
|
|
|
5
5
|
|
|
6
|
+
class ToolOutput(BaseModel):
|
|
7
|
+
result: str
|
|
8
|
+
analysis: str
|
|
9
|
+
logprobs: list[dict[str, Any]]
|
|
10
|
+
errors: list[str]
|
|
11
|
+
|
|
12
|
+
|
|
6
13
|
class StrOutput(BaseModel):
|
|
7
14
|
result: str = Field(..., description="The output string")
|
|
8
15
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/hamtaa_texttools.egg-info/dependency_links.txt
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/prompts/subject_to_question.yaml
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{hamtaa_texttools-1.0.8 → hamtaa_texttools-1.1.0}/texttools/tools/internals/prompt_loader.py
RENAMED
|
File without changes
|
|
File without changes
|