hamtaa-texttools 0.1.48__tar.gz → 0.1.49__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/PKG-INFO +1 -1
  2. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/hamtaa_texttools.egg-info/PKG-INFO +1 -1
  3. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/pyproject.toml +1 -1
  4. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/utils/batch_manager/batch_runner.py +1 -1
  5. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/utils/flex_processor.py +31 -29
  6. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/README.md +0 -0
  7. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/hamtaa_texttools.egg-info/SOURCES.txt +0 -0
  8. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/hamtaa_texttools.egg-info/dependency_links.txt +0 -0
  9. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/hamtaa_texttools.egg-info/requires.txt +0 -0
  10. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/hamtaa_texttools.egg-info/top_level.txt +0 -0
  11. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/setup.cfg +0 -0
  12. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/setup.py +0 -0
  13. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/tests/test_vllm_output.py +0 -0
  14. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/__init__.py +0 -0
  15. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/__init__.py +0 -0
  16. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_categorizer.py +0 -0
  17. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_keyword_extractor.py +0 -0
  18. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_ner_extractor.py +0 -0
  19. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_question_detector.py +0 -0
  20. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_question_generator.py +0 -0
  21. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_question_merger.py +0 -0
  22. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_question_rewriter.py +0 -0
  23. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_router.py +0 -0
  24. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_summarizer.py +0 -0
  25. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_task_performer.py +0 -0
  26. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/base/base_translator.py +0 -0
  27. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/formatter/__init__.py +0 -0
  28. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/formatter/base.py +0 -0
  29. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/formatter/gemma3_formatter.py +0 -0
  30. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/handlers/__init__.py +0 -0
  31. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/handlers/categorizer/__init__.py +0 -0
  32. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/handlers/categorizer/categorizer.py +0 -0
  33. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/handlers/handlers.py +0 -0
  34. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/__init__.py +0 -0
  35. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/categorizer/__init__.py +0 -0
  36. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/categorizer/encoder_model/__init__.py +0 -0
  37. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/categorizer/encoder_model/encoder_vectorizer.py +0 -0
  38. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/categorizer/llm/__init__.py +0 -0
  39. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/categorizer/llm/gemma_categorizer.py +0 -0
  40. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/categorizer/llm/openai_categorizer.py +0 -0
  41. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/keyword_extractor/__init__.py +0 -0
  42. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/keyword_extractor/gemma_extractor.py +0 -0
  43. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/merger/__init__.py +0 -0
  44. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/merger/gemma_question_merger.py +0 -0
  45. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/ner/__init__.py +0 -0
  46. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/ner/gemma_ner_extractor.py +0 -0
  47. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/question_detector/__init__.py +0 -0
  48. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/question_detector/gemma_detector.py +0 -0
  49. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/question_detector/llm_detector.py +0 -0
  50. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/question_generator/__init__.py +0 -0
  51. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/question_generator/gemma_question_generator.py +0 -0
  52. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/reranker/__init__.py +0 -0
  53. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/reranker/reranker.py +0 -0
  54. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/reranker/scorer.py +0 -0
  55. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/reranker/sorter.py +0 -0
  56. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/rewriter/__init__.py +0 -0
  57. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/rewriter/gemma_question_rewriter.py +0 -0
  58. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/router/__init__.py +0 -0
  59. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/router/gemma_router.py +0 -0
  60. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/subject_to_question/__init__.py +0 -0
  61. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/subject_to_question/gemma_question_generator.py +0 -0
  62. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/summarizer/__init__.py +0 -0
  63. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/summarizer/gemma_summarizer.py +0 -0
  64. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/summarizer/llm_summerizer.py +0 -0
  65. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/translator/__init__.py +0 -0
  66. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/tools/translator/gemma_translator.py +0 -0
  67. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/utils/batch_manager/__init__.py +0 -0
  68. {hamtaa_texttools-0.1.48 → hamtaa_texttools-0.1.49}/texttools/utils/batch_manager/batch_manager.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hamtaa-texttools
3
- Version: 0.1.48
3
+ Version: 0.1.49
4
4
  Summary: A set of high-level NLP tools
5
5
  Author: Tohidi, Montazer, Givechi, Mousavinezhad
6
6
  Requires-Python: >=3.8
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: hamtaa-texttools
3
- Version: 0.1.48
3
+ Version: 0.1.49
4
4
  Summary: A set of high-level NLP tools
5
5
  Author: Tohidi, Montazer, Givechi, Mousavinezhad
6
6
  Requires-Python: >=3.8
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "hamtaa-texttools"
7
- version = "0.1.48"
7
+ version = "0.1.49"
8
8
  description = "A set of high-level NLP tools"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.8"
@@ -9,7 +9,7 @@ from typing import Any, Callable
9
9
  from openai import OpenAI
10
10
  from pydantic import BaseModel
11
11
 
12
- from texttools.batch_manager import SimpleBatchManager
12
+ from texttools import SimpleBatchManager
13
13
 
14
14
 
15
15
  class OutputModel(BaseModel):
@@ -1,23 +1,25 @@
1
- import random
2
1
  import asyncio
3
- from openai import OpenAI, RateLimitError, APIError
2
+ import random
4
3
  from typing import Optional
4
+
5
+ from openai import APIError, OpenAI, RateLimitError
5
6
  from pydantic import BaseModel, ValidationError
6
- import httpx
7
7
 
8
8
  # http_client = httpx()
9
9
  # test_client = OpenAI(http_client=http_client)
10
10
 
11
+
11
12
  async def flex_processing(
12
- LLM_client: OpenAI,
13
- system_prompt: str,
14
- user_prompt: str,
15
- output_model: Optional[BaseModel]=None,
16
- prompt_cache_key: Optional[str]=None,
17
- max_retries: int = 10,
18
- base_delay: float = 2.0,
19
- model_name: Optional[str] ="gpt-5-mini",
20
- **client_kwargs):
13
+ LLM_client: OpenAI,
14
+ system_prompt: str,
15
+ user_prompt: str,
16
+ output_model: Optional[BaseModel] = None,
17
+ prompt_cache_key: Optional[str] = None,
18
+ max_retries: int = 10,
19
+ base_delay: float = 2.0,
20
+ model_name: Optional[str] = "gpt-5-mini",
21
+ **client_kwargs,
22
+ ):
21
23
  """
22
24
  Wrapper for flex processing with retry and exponential backoff.
23
25
  Handles 429 'Resource Unavailable' errors gracefully.
@@ -25,24 +27,24 @@ async def flex_processing(
25
27
  for attempt in range(max_retries):
26
28
  try:
27
29
  request_kwargs = {
28
- "model": model_name,
29
- "messages": [
30
- {"role": "system", "content": system_prompt},
31
- {"role": "user", "content": user_prompt},
32
- ],
33
- "service_tier": "flex",
34
- "timeout": 900.0,
35
- **client_kwargs
36
- }
30
+ "model": model_name,
31
+ "messages": [
32
+ {"role": "system", "content": system_prompt},
33
+ {"role": "user", "content": user_prompt},
34
+ ],
35
+ "service_tier": "flex",
36
+ "timeout": 900.0,
37
+ **client_kwargs,
38
+ }
37
39
  if output_model:
38
40
  request_kwargs["response_format"] = output_model
39
41
  if prompt_cache_key:
40
42
  request_kwargs["prompt_cache_key"] = prompt_cache_key
41
-
43
+
42
44
  response = LLM_client.chat.completions.parse(**request_kwargs)
43
45
  # response = self.client.chat.completions.parse(output_model)
44
46
  content = response.choices[0].message.content
45
- # ✅ Validate structured output if a model is provided
47
+ # ✅ Validate structured output if a model is provided
46
48
  if output_model is not None:
47
49
  try:
48
50
  output_model.model_validate_json(content)
@@ -51,17 +53,17 @@ async def flex_processing(
51
53
  return base_content
52
54
  except ValidationError as ve:
53
55
  # Treat invalid output as retryable
54
- wait_time = base_delay * (2 ** attempt) + random.uniform(0, 1)
56
+ wait_time = base_delay * (2**attempt) + random.uniform(0, 1)
55
57
  print(
56
- f"[Flex Retry] Attempt {attempt+1}/{max_retries} produced invalid structured output. "
58
+ f"[Flex Retry] Attempt {attempt + 1}/{max_retries} produced invalid structured output. "
57
59
  f"Retrying in {wait_time:.2f}s... (ValidationError: {ve})"
58
60
  )
59
61
  await asyncio.sleep(wait_time)
60
- continue
62
+ continue
61
63
  except (RateLimitError, APIError) as e:
62
- wait_time = base_delay * (2 ** attempt) + random.uniform(0, 1)
64
+ wait_time = base_delay * (2**attempt) + random.uniform(0, 1)
63
65
  print(
64
- f"[Flex Retry] Attempt {attempt+1}/{max_retries} failed "
66
+ f"[Flex Retry] Attempt {attempt + 1}/{max_retries} failed "
65
67
  f"with error: {type(e).__name__} - {e}. "
66
68
  f"Retrying in {wait_time:.2f}s..."
67
69
  )
@@ -75,4 +77,4 @@ async def flex_processing(
75
77
 
76
78
  raise RuntimeError(
77
79
  f"[Flex Processing] Exhausted {max_retries} retries for prompt_key={prompt_cache_key}"
78
- )
80
+ )