retab 0.0.37__py3-none-any.whl → 0.0.39__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- retab/__init__.py +2 -2
- retab/_resource.py +5 -5
- retab/_utils/_model_cards/anthropic.yaml +59 -0
- retab/_utils/_model_cards/auto.yaml +43 -0
- retab/_utils/_model_cards/gemini.yaml +117 -0
- retab/_utils/_model_cards/openai.yaml +301 -0
- retab/_utils/_model_cards/xai.yaml +28 -0
- retab/_utils/ai_models.py +109 -71
- retab/_utils/chat.py +20 -20
- retab/_utils/responses.py +14 -14
- retab/_utils/usage/usage.py +5 -4
- retab/client.py +22 -22
- retab/resources/consensus/client.py +2 -2
- retab/resources/consensus/completions.py +26 -26
- retab/resources/consensus/completions_stream.py +27 -27
- retab/resources/consensus/responses.py +11 -11
- retab/resources/consensus/responses_stream.py +15 -15
- retab/resources/documents/client.py +297 -16
- retab/resources/documents/extractions.py +39 -39
- retab/resources/evaluations/documents.py +5 -5
- retab/resources/evaluations/iterations.py +7 -7
- retab/resources/jsonlUtils.py +7 -7
- retab/resources/processors/automations/endpoints.py +2 -2
- retab/resources/processors/automations/links.py +2 -2
- retab/resources/processors/automations/logs.py +2 -2
- retab/resources/processors/automations/mailboxes.py +2 -2
- retab/resources/processors/automations/outlook.py +2 -2
- retab/resources/processors/client.py +9 -9
- retab/resources/usage.py +4 -4
- retab/types/ai_models.py +41 -513
- retab/types/automations/mailboxes.py +1 -1
- retab/types/automations/webhooks.py +3 -3
- retab/types/chat.py +1 -1
- retab/types/completions.py +10 -10
- retab/types/documents/__init__.py +3 -0
- retab/types/documents/create_messages.py +2 -2
- retab/types/documents/extractions.py +19 -19
- retab/types/documents/parse.py +32 -0
- retab/types/extractions.py +4 -4
- retab/types/logs.py +2 -2
- retab/types/schemas/object.py +3 -3
- {retab-0.0.37.dist-info → retab-0.0.39.dist-info}/METADATA +72 -72
- {retab-0.0.37.dist-info → retab-0.0.39.dist-info}/RECORD +45 -39
- {retab-0.0.37.dist-info → retab-0.0.39.dist-info}/WHEEL +0 -0
- {retab-0.0.37.dist-info → retab-0.0.39.dist-info}/top_level.txt +0 -0
retab/_utils/ai_models.py
CHANGED
@@ -1,74 +1,121 @@
|
|
1
|
+
import os
|
2
|
+
import yaml
|
1
3
|
from typing import get_args
|
2
4
|
|
3
|
-
from ..types.ai_models import AIProvider, GeminiModel, OpenAIModel, xAI_Model
|
5
|
+
from ..types.ai_models import AIProvider, GeminiModel, OpenAIModel, xAI_Model, RetabModel, PureLLMModel, ModelCard
|
4
6
|
|
7
|
+
MODEL_CARDS_DIR = os.path.join(os.path.dirname(__file__), "_model_cards")
|
5
8
|
|
6
|
-
def
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
elif model in get_args(xAI_Model):
|
17
|
-
return "xAI"
|
18
|
-
elif model in get_args(GeminiModel):
|
19
|
-
return "Gemini"
|
20
|
-
raise ValueError(f"Could not determine AI provider for model: {model}")
|
9
|
+
def merge_model_cards(base: dict, override: dict) -> dict:
|
10
|
+
result = base.copy()
|
11
|
+
for key, value in override.items():
|
12
|
+
if key == "inherits":
|
13
|
+
continue
|
14
|
+
if isinstance(value, dict) and key in result:
|
15
|
+
result[key] = merge_model_cards(result[key], value)
|
16
|
+
else:
|
17
|
+
result[key] = value
|
18
|
+
return result
|
21
19
|
|
20
|
+
def load_model_cards(yaml_file: str) -> list[ModelCard]:
|
21
|
+
raw_cards = yaml.safe_load(open(yaml_file))
|
22
|
+
name_to_card = {c["model"]: c for c in raw_cards if "inherits" not in c}
|
22
23
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
# return
|
33
|
-
elif model in get_args(xAI_Model):
|
34
|
-
return
|
35
|
-
elif model in get_args(GeminiModel):
|
36
|
-
return
|
37
|
-
raise ValueError(
|
38
|
-
f"Invalid model for extraction: {model}.\nValid OpenAI models: {get_args(OpenAIModel)}\n"
|
39
|
-
# f"Valid Anthropic models: {get_args(AnthropicModel)}\n"
|
40
|
-
# f"Valid xAI models: {get_args(xAI_Model)}\n"
|
41
|
-
# f"Valid Gemini models: {get_args(GeminiModel)}"
|
42
|
-
)
|
24
|
+
final_cards = []
|
25
|
+
for card in raw_cards:
|
26
|
+
if "inherits" in card:
|
27
|
+
parent = name_to_card[card["inherits"]]
|
28
|
+
merged = merge_model_cards(parent, card)
|
29
|
+
final_cards.append(ModelCard(**merged))
|
30
|
+
else:
|
31
|
+
final_cards.append(ModelCard(**card))
|
32
|
+
return final_cards
|
43
33
|
|
34
|
+
# Load all model cards
|
35
|
+
model_cards = sum([
|
36
|
+
load_model_cards(os.path.join(MODEL_CARDS_DIR, "openai.yaml")),
|
37
|
+
load_model_cards(os.path.join(MODEL_CARDS_DIR, "anthropic.yaml")),
|
38
|
+
load_model_cards(os.path.join(MODEL_CARDS_DIR, "xai.yaml")),
|
39
|
+
load_model_cards(os.path.join(MODEL_CARDS_DIR, "gemini.yaml")),
|
40
|
+
load_model_cards(os.path.join(MODEL_CARDS_DIR, "auto.yaml")),
|
41
|
+
], [])
|
42
|
+
model_cards_dict = {card.model: card for card in model_cards}
|
44
43
|
|
45
|
-
def assert_valid_model_batch_processing(model: str) -> None:
|
46
|
-
"""Assert that the model is either a standard OpenAI model or a valid fine-tuned model.
|
47
44
|
|
48
|
-
|
49
|
-
|
50
|
-
|
45
|
+
# Validate that model cards
|
46
|
+
all_model_names = set(model_cards_dict.keys())
|
47
|
+
if all_model_names.symmetric_difference(set(get_args(PureLLMModel))):
|
48
|
+
raise ValueError(f"Mismatch between model cards and PureLLMModel type: {all_model_names.symmetric_difference(set(get_args(PureLLMModel)))}")
|
49
|
+
|
50
|
+
|
51
|
+
def get_model_from_model_id(model_id: str) -> str:
|
52
|
+
"""
|
53
|
+
Get the model name from the model id.
|
54
|
+
"""
|
55
|
+
if model_id.startswith("ft:"):
|
56
|
+
parts = model_id.split(":")
|
57
|
+
return parts[1]
|
58
|
+
else:
|
59
|
+
return model_id
|
60
|
+
|
61
|
+
|
62
|
+
def get_model_card(model: str) -> ModelCard:
|
63
|
+
"""
|
64
|
+
Get the model card for a specific model.
|
65
|
+
|
66
|
+
Args:
|
67
|
+
model: The model name to look up
|
68
|
+
|
69
|
+
Returns:
|
70
|
+
The ModelCard for the specified model
|
51
71
|
|
52
72
|
Raises:
|
53
|
-
ValueError: If
|
73
|
+
ValueError: If no model card is found for the specified model
|
54
74
|
"""
|
55
|
-
|
56
|
-
|
75
|
+
model_name = get_model_from_model_id(model)
|
76
|
+
if model_name in model_cards_dict:
|
77
|
+
model_card = ModelCard(**model_cards_dict[model_name].model_dump())
|
78
|
+
if model_name != model:
|
79
|
+
# Fine-tuned model -> Change the name
|
80
|
+
model_card.model = model
|
81
|
+
# Remove the fine-tuning feature (if exists)
|
82
|
+
try:
|
83
|
+
model_card.capabilities.features.remove("fine_tuning")
|
84
|
+
except ValueError:
|
85
|
+
pass
|
86
|
+
return model_card
|
87
|
+
|
88
|
+
raise ValueError(f"No model card found for model: {model_name}")
|
57
89
|
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
90
|
+
|
91
|
+
def get_provider_for_model(model_id: str) -> AIProvider:
|
92
|
+
"""
|
93
|
+
Determine the AI provider associated with the given model identifier.
|
94
|
+
Returns one of: "Anthropic", "xAI", "OpenAI", "Gemini", "Retab" or None if unknown.
|
95
|
+
"""
|
96
|
+
model_name = get_model_from_model_id(model_id)
|
97
|
+
# if model_name in get_args(AnthropicModel):
|
98
|
+
# return "Anthropic"
|
99
|
+
# if model_name in get_args(xAI_Model):
|
100
|
+
# return "xAI"
|
101
|
+
if model_name in get_args(OpenAIModel):
|
102
|
+
return "OpenAI"
|
103
|
+
if model_name in get_args(GeminiModel):
|
104
|
+
return "Gemini"
|
105
|
+
if model_name in get_args(RetabModel):
|
106
|
+
return "Retab"
|
107
|
+
raise ValueError(f"Unknown model: {model_name}")
|
108
|
+
|
109
|
+
|
110
|
+
def assert_valid_model_extraction(model: str) -> None:
|
111
|
+
try:
|
112
|
+
get_provider_for_model(model)
|
64
113
|
except ValueError:
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
) from None
|
71
|
-
raise
|
114
|
+
raise ValueError(
|
115
|
+
f"Invalid model for extraction: {model}.\nValid OpenAI models: {get_args(OpenAIModel)}\n"
|
116
|
+
f"Valid xAI models: {get_args(xAI_Model)}\n"
|
117
|
+
f"Valid Gemini models: {get_args(GeminiModel)}"
|
118
|
+
) from None
|
72
119
|
|
73
120
|
|
74
121
|
def assert_valid_model_schema_generation(model: str) -> None:
|
@@ -81,20 +128,11 @@ def assert_valid_model_schema_generation(model: str) -> None:
|
|
81
128
|
Raises:
|
82
129
|
ValueError: If the model format is invalid
|
83
130
|
"""
|
84
|
-
if model in get_args(OpenAIModel):
|
131
|
+
if get_model_from_model_id(model) in get_args(OpenAIModel):
|
85
132
|
return
|
86
|
-
|
87
|
-
|
88
|
-
ft, base_model, model_id = model.split(":", 2)
|
89
|
-
if base_model not in get_args(OpenAIModel):
|
90
|
-
raise ValueError(f"Invalid base model in fine-tuned model '{model}'. Base model must be one of: {get_args(OpenAIModel)}")
|
91
|
-
if not model_id or not model_id.strip():
|
92
|
-
raise ValueError(f"Model ID cannot be empty in fine-tuned model '{model}'")
|
93
|
-
except ValueError:
|
94
|
-
if ":" not in model:
|
95
|
-
raise ValueError(
|
133
|
+
else:
|
134
|
+
raise ValueError(
|
96
135
|
f"Invalid model format: {model}. Must be either:\n"
|
97
136
|
f"1. A standard model: {get_args(OpenAIModel)}\n"
|
98
|
-
f"2. A fine-tuned model in format 'base_model:id' where base_model is one of the standard models"
|
99
|
-
) from None
|
100
|
-
raise
|
137
|
+
f"2. A fine-tuned model in format 'base_model:id' where base_model is one of the standard openai models"
|
138
|
+
) from None
|
retab/_utils/chat.py
CHANGED
@@ -13,14 +13,14 @@ from openai.types.chat.chat_completion_content_part_param import ChatCompletionC
|
|
13
13
|
from openai.types.chat.chat_completion_content_part_text_param import ChatCompletionContentPartTextParam
|
14
14
|
from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
|
15
15
|
|
16
|
-
from ..types.chat import
|
16
|
+
from ..types.chat import ChatCompletionRetabMessage
|
17
17
|
|
18
18
|
MediaType = Literal["image/jpeg", "image/png", "image/gif", "image/webp"]
|
19
19
|
|
20
20
|
|
21
|
-
def convert_to_google_genai_format(messages: List[
|
21
|
+
def convert_to_google_genai_format(messages: List[ChatCompletionRetabMessage]) -> tuple[str, list[ContentUnionDict]]:
|
22
22
|
"""
|
23
|
-
Converts a list of
|
23
|
+
Converts a list of ChatCompletionRetabMessage to a format compatible with the google.genai SDK.
|
24
24
|
|
25
25
|
|
26
26
|
Example:
|
@@ -40,7 +40,7 @@ def convert_to_google_genai_format(messages: List[ChatCompletionUiformMessage])
|
|
40
40
|
```
|
41
41
|
|
42
42
|
Args:
|
43
|
-
messages (List[
|
43
|
+
messages (List[ChatCompletionRetabMessage]): List of chat messages.
|
44
44
|
|
45
45
|
Returns:
|
46
46
|
List[Union[Dict[str, str], str]]: A list of formatted inputs for the google.genai SDK.
|
@@ -94,12 +94,12 @@ def convert_to_google_genai_format(messages: List[ChatCompletionUiformMessage])
|
|
94
94
|
return system_message, formatted_content
|
95
95
|
|
96
96
|
|
97
|
-
def convert_to_anthropic_format(messages: List[
|
97
|
+
def convert_to_anthropic_format(messages: List[ChatCompletionRetabMessage]) -> tuple[str, List[MessageParam]]:
|
98
98
|
"""
|
99
|
-
Converts a list of
|
99
|
+
Converts a list of ChatCompletionRetabMessage to a format compatible with the Anthropic SDK.
|
100
100
|
|
101
101
|
Args:
|
102
|
-
messages (List[
|
102
|
+
messages (List[ChatCompletionRetabMessage]): List of chat messages.
|
103
103
|
|
104
104
|
Returns:
|
105
105
|
(system_message, formatted_messages):
|
@@ -216,11 +216,11 @@ def convert_to_anthropic_format(messages: List[ChatCompletionUiformMessage]) ->
|
|
216
216
|
return system_message, formatted_messages
|
217
217
|
|
218
218
|
|
219
|
-
def convert_from_anthropic_format(messages: list[MessageParam], system_prompt: str) -> list[
|
219
|
+
def convert_from_anthropic_format(messages: list[MessageParam], system_prompt: str) -> list[ChatCompletionRetabMessage]:
|
220
220
|
"""
|
221
|
-
Converts a list of Anthropic MessageParam to a list of
|
221
|
+
Converts a list of Anthropic MessageParam to a list of ChatCompletionRetabMessage.
|
222
222
|
"""
|
223
|
-
formatted_messages: list[
|
223
|
+
formatted_messages: list[ChatCompletionRetabMessage] = [ChatCompletionRetabMessage(role="developer", content=system_prompt)]
|
224
224
|
|
225
225
|
for message in messages:
|
226
226
|
role = message["role"]
|
@@ -229,7 +229,7 @@ def convert_from_anthropic_format(messages: list[MessageParam], system_prompt: s
|
|
229
229
|
# Handle different content structures
|
230
230
|
if isinstance(content_blocks, list) and len(content_blocks) == 1 and isinstance(content_blocks[0], dict) and content_blocks[0].get("type") == "text":
|
231
231
|
# Simple text message
|
232
|
-
formatted_messages.append(cast(
|
232
|
+
formatted_messages.append(cast(ChatCompletionRetabMessage, {"role": role, "content": content_blocks[0].get("text", "")}))
|
233
233
|
elif isinstance(content_blocks, list):
|
234
234
|
# Message with multiple content parts or non-text content
|
235
235
|
formatted_content: list[ChatCompletionContentPartParam] = []
|
@@ -248,22 +248,22 @@ def convert_from_anthropic_format(messages: list[MessageParam], system_prompt: s
|
|
248
248
|
|
249
249
|
formatted_content.append(cast(ChatCompletionContentPartParam, {"type": "image_url", "image_url": {"url": image_url}}))
|
250
250
|
|
251
|
-
formatted_messages.append(cast(
|
251
|
+
formatted_messages.append(cast(ChatCompletionRetabMessage, {"role": role, "content": formatted_content}))
|
252
252
|
|
253
253
|
return formatted_messages
|
254
254
|
|
255
255
|
|
256
|
-
def convert_to_openai_format(messages: List[
|
256
|
+
def convert_to_openai_format(messages: List[ChatCompletionRetabMessage]) -> List[ChatCompletionMessageParam]:
|
257
257
|
return cast(list[ChatCompletionMessageParam], messages)
|
258
258
|
|
259
259
|
|
260
|
-
def convert_from_openai_format(messages: list[ChatCompletionMessageParam]) -> list[
|
261
|
-
return cast(list[
|
260
|
+
def convert_from_openai_format(messages: list[ChatCompletionMessageParam]) -> list[ChatCompletionRetabMessage]:
|
261
|
+
return cast(list[ChatCompletionRetabMessage], messages)
|
262
262
|
|
263
263
|
|
264
264
|
def separate_messages(
|
265
|
-
messages: list[
|
266
|
-
) -> tuple[Optional[
|
265
|
+
messages: list[ChatCompletionRetabMessage],
|
266
|
+
) -> tuple[Optional[ChatCompletionRetabMessage], list[ChatCompletionRetabMessage], list[ChatCompletionRetabMessage]]:
|
267
267
|
"""
|
268
268
|
Separates messages into system, user and assistant messages.
|
269
269
|
|
@@ -291,12 +291,12 @@ def separate_messages(
|
|
291
291
|
return system_message, user_messages, assistant_messages
|
292
292
|
|
293
293
|
|
294
|
-
def str_messages(messages: list[
|
294
|
+
def str_messages(messages: list[ChatCompletionRetabMessage], max_length: int = 100) -> str:
|
295
295
|
"""
|
296
296
|
Converts a list of chat messages into a string representation with faithfully serialized structure.
|
297
297
|
|
298
298
|
Args:
|
299
|
-
messages (list[
|
299
|
+
messages (list[ChatCompletionRetabMessage]): The list of chat messages.
|
300
300
|
max_length (int): Maximum length for content before truncation.
|
301
301
|
|
302
302
|
Returns:
|
@@ -307,7 +307,7 @@ def str_messages(messages: list[ChatCompletionUiformMessage], max_length: int =
|
|
307
307
|
"""Truncate text to max_len with ellipsis."""
|
308
308
|
return text if len(text) <= max_len else f"{text[:max_len]}..."
|
309
309
|
|
310
|
-
serialized: list[
|
310
|
+
serialized: list[ChatCompletionRetabMessage] = []
|
311
311
|
for message in messages:
|
312
312
|
role = message["role"]
|
313
313
|
content = message["content"]
|
retab/_utils/responses.py
CHANGED
@@ -16,13 +16,13 @@ from openai.types.responses.response_input_message_content_list_param import Res
|
|
16
16
|
from openai.types.responses.response_input_param import ResponseInputItemParam
|
17
17
|
from openai.types.responses.response_input_text_param import ResponseInputTextParam
|
18
18
|
|
19
|
-
from ..types.chat import
|
20
|
-
from ..types.documents.extractions import
|
19
|
+
from ..types.chat import ChatCompletionRetabMessage
|
20
|
+
from ..types.documents.extractions import RetabParsedChatCompletion, RetabParsedChoice
|
21
21
|
|
22
22
|
|
23
|
-
def convert_to_openai_format(messages: list[
|
23
|
+
def convert_to_openai_format(messages: list[ChatCompletionRetabMessage]) -> list[ResponseInputItemParam]:
|
24
24
|
"""
|
25
|
-
Converts a list of
|
25
|
+
Converts a list of ChatCompletionRetabMessage to the OpenAI ResponseInputParam format.
|
26
26
|
|
27
27
|
Args:
|
28
28
|
messages: List of chat messages in UIForm format
|
@@ -64,9 +64,9 @@ def convert_to_openai_format(messages: list[ChatCompletionUiformMessage]) -> lis
|
|
64
64
|
return formatted_messages
|
65
65
|
|
66
66
|
|
67
|
-
def convert_from_openai_format(messages: list[ResponseInputItemParam]) -> list[
|
67
|
+
def convert_from_openai_format(messages: list[ResponseInputItemParam]) -> list[ChatCompletionRetabMessage]:
|
68
68
|
"""
|
69
|
-
Converts messages from OpenAI ResponseInputParam format to
|
69
|
+
Converts messages from OpenAI ResponseInputParam format to ChatCompletionRetabMessage format.
|
70
70
|
|
71
71
|
Args:
|
72
72
|
messages: Messages in OpenAI ResponseInputParam format
|
@@ -74,7 +74,7 @@ def convert_from_openai_format(messages: list[ResponseInputItemParam]) -> list[C
|
|
74
74
|
Returns:
|
75
75
|
List of chat messages in UIForm format
|
76
76
|
"""
|
77
|
-
formatted_messages: list[
|
77
|
+
formatted_messages: list[ChatCompletionRetabMessage] = []
|
78
78
|
|
79
79
|
for message in messages:
|
80
80
|
if "role" not in message or "content" not in message:
|
@@ -110,23 +110,23 @@ def convert_from_openai_format(messages: list[ResponseInputItemParam]) -> list[C
|
|
110
110
|
print(f"Not supported content type: {part['type']}... Skipping...")
|
111
111
|
|
112
112
|
# Create message in UIForm format
|
113
|
-
formatted_message =
|
113
|
+
formatted_message = ChatCompletionRetabMessage(role=role, content=formatted_content)
|
114
114
|
formatted_messages.append(formatted_message)
|
115
115
|
|
116
116
|
return formatted_messages
|
117
117
|
|
118
118
|
|
119
|
-
def parse_openai_responses_response(response: Response) ->
|
119
|
+
def parse_openai_responses_response(response: Response) -> RetabParsedChatCompletion:
|
120
120
|
"""
|
121
|
-
Convert an OpenAI Response (Responses API) to
|
121
|
+
Convert an OpenAI Response (Responses API) to RetabParsedChatCompletion type.
|
122
122
|
|
123
123
|
Args:
|
124
124
|
response: Response from OpenAI Responses API
|
125
125
|
|
126
126
|
Returns:
|
127
|
-
Parsed response in
|
127
|
+
Parsed response in RetabParsedChatCompletion format
|
128
128
|
"""
|
129
|
-
# Create the
|
129
|
+
# Create the RetabParsedChatCompletion object
|
130
130
|
if response.usage:
|
131
131
|
usage = CompletionUsage(
|
132
132
|
prompt_tokens=response.usage.input_tokens,
|
@@ -148,7 +148,7 @@ def parse_openai_responses_response(response: Response) -> UiParsedChatCompletio
|
|
148
148
|
result_object = from_json(bytes(output_text, "utf-8"), partial_mode=True) # Attempt to parse the result even if EOF is reached
|
149
149
|
|
150
150
|
choices.append(
|
151
|
-
|
151
|
+
RetabParsedChoice(
|
152
152
|
index=0,
|
153
153
|
message=ParsedChatCompletionMessage(
|
154
154
|
role="assistant",
|
@@ -158,7 +158,7 @@ def parse_openai_responses_response(response: Response) -> UiParsedChatCompletio
|
|
158
158
|
)
|
159
159
|
)
|
160
160
|
|
161
|
-
return
|
161
|
+
return RetabParsedChatCompletion(
|
162
162
|
id=response.id,
|
163
163
|
choices=choices,
|
164
164
|
created=int(datetime.datetime.now().timestamp()),
|
retab/_utils/usage/usage.py
CHANGED
@@ -4,7 +4,8 @@ from openai.types.completion_usage import CompletionUsage
|
|
4
4
|
from pydantic import BaseModel, Field
|
5
5
|
|
6
6
|
# https://platform.openai.com/docs/guides/prompt-caching
|
7
|
-
from ...types.ai_models import Amount, Pricing
|
7
|
+
from ...types.ai_models import Amount, Pricing
|
8
|
+
from ..._utils.ai_models import get_model_card
|
8
9
|
|
9
10
|
# ─── PRICING MODELS ────────────────────────────────────────────────────────────
|
10
11
|
|
@@ -78,7 +79,7 @@ def compute_api_call_cost(pricing: Pricing, usage: CompletionUsage, is_ft: bool
|
|
78
79
|
|
79
80
|
|
80
81
|
def compute_cost_from_model(model: str, usage: CompletionUsage) -> Amount:
|
81
|
-
# Extract base model name for fine-tuned models like "ft:gpt-4o:
|
82
|
+
# Extract base model name for fine-tuned models like "ft:gpt-4o:retab:4389573"
|
82
83
|
is_ft = False
|
83
84
|
if model.startswith("ft:"):
|
84
85
|
# Split by colon and take the second part (index 1) which contains the base model
|
@@ -270,7 +271,7 @@ def compute_cost_from_model_with_breakdown(model: str, usage: CompletionUsage) -
|
|
270
271
|
Computes a detailed cost breakdown for an API call using the specified model and usage.
|
271
272
|
|
272
273
|
Args:
|
273
|
-
model: The model name (can be a fine-tuned model like "ft:gpt-4o:
|
274
|
+
model: The model name (can be a fine-tuned model like "ft:gpt-4o:retab:4389573")
|
274
275
|
usage: Token usage statistics for the API call
|
275
276
|
|
276
277
|
Returns:
|
@@ -279,7 +280,7 @@ def compute_cost_from_model_with_breakdown(model: str, usage: CompletionUsage) -
|
|
279
280
|
Raises:
|
280
281
|
ValueError: If no pricing information is found for the model
|
281
282
|
"""
|
282
|
-
# Extract base model name for fine-tuned models like "ft:gpt-4o:
|
283
|
+
# Extract base model name for fine-tuned models like "ft:gpt-4o:retab:4389573"
|
283
284
|
original_model = model
|
284
285
|
is_ft = False
|
285
286
|
|
retab/client.py
CHANGED
@@ -26,15 +26,15 @@ def raise_max_tries_exceeded(details: backoff.types.Details) -> None:
|
|
26
26
|
raise Exception(f"Max tries exceeded after {tries} tries.")
|
27
27
|
|
28
28
|
|
29
|
-
class
|
30
|
-
"""Base class for
|
29
|
+
class BaseRetab:
|
30
|
+
"""Base class for Retab clients that handles authentication and configuration.
|
31
31
|
|
32
32
|
This class provides core functionality for API authentication, configuration, and common HTTP operations
|
33
33
|
used by both synchronous and asynchronous clients.
|
34
34
|
|
35
35
|
Args:
|
36
|
-
api_key (str, optional):
|
37
|
-
base_url (str, optional): Base URL for API requests. Defaults to https://api.
|
36
|
+
api_key (str, optional): Retab API key. If not provided, will look for RETAB_API_KEY env variable.
|
37
|
+
base_url (str, optional): Base URL for API requests. Defaults to https://api.retab.dev
|
38
38
|
timeout (float): Request timeout in seconds. Defaults to 240.0
|
39
39
|
max_retries (int): Maximum number of retries for failed requests. Defaults to 3
|
40
40
|
openai_api_key (str, optional): OpenAI API key. Will look for OPENAI_API_KEY env variable if not provided
|
@@ -59,16 +59,16 @@ class BaseUiForm:
|
|
59
59
|
xai_api_key: Optional[str] = PydanticUndefined, # type: ignore[assignment]
|
60
60
|
) -> None:
|
61
61
|
if api_key is None:
|
62
|
-
api_key = os.environ.get("
|
62
|
+
api_key = os.environ.get("RETAB_API_KEY")
|
63
63
|
|
64
64
|
if api_key is None:
|
65
65
|
raise ValueError(
|
66
|
-
"No API key provided. You can create an API key at https://
|
67
|
-
"Then either pass it to the client (api_key='your-key') or set the
|
66
|
+
"No API key provided. You can create an API key at https://retab.dev\n"
|
67
|
+
"Then either pass it to the client (api_key='your-key') or set the RETAB_API_KEY environment variable"
|
68
68
|
)
|
69
69
|
|
70
70
|
if base_url is None:
|
71
|
-
base_url = os.environ.get("
|
71
|
+
base_url = os.environ.get("RETAB_API_BASE_URL", "https://api.retab.dev")
|
72
72
|
|
73
73
|
truststore.inject_into_ssl()
|
74
74
|
self.api_key = api_key
|
@@ -146,15 +146,15 @@ class BaseUiForm:
|
|
146
146
|
return response.text
|
147
147
|
|
148
148
|
|
149
|
-
class
|
150
|
-
"""Synchronous client for interacting with the
|
149
|
+
class Retab(BaseRetab):
|
150
|
+
"""Synchronous client for interacting with the Retab API.
|
151
151
|
|
152
|
-
This client provides synchronous access to all
|
152
|
+
This client provides synchronous access to all Retab API resources including files, fine-tuning,
|
153
153
|
prompt optimization, documents, models, datasets, and schemas.
|
154
154
|
|
155
155
|
Args:
|
156
|
-
api_key (str, optional):
|
157
|
-
base_url (str, optional): Base URL for API requests. Defaults to https://api.
|
156
|
+
api_key (str, optional): Retab API key. If not provided, will look for RETAB_API_KEY env variable.
|
157
|
+
base_url (str, optional): Base URL for API requests. Defaults to https://api.retab.dev
|
158
158
|
timeout (float): Request timeout in seconds. Defaults to 240.0
|
159
159
|
max_retries (int): Maximum number of retries for failed requests. Defaults to 3
|
160
160
|
openai_api_key (str, optional): OpenAI API key. Will look for OPENAI_API_KEY env variable if not provided
|
@@ -395,11 +395,11 @@ class UiForm(BaseUiForm):
|
|
395
395
|
"""Closes the HTTP client session."""
|
396
396
|
self.client.close()
|
397
397
|
|
398
|
-
def __enter__(self) -> "
|
398
|
+
def __enter__(self) -> "Retab":
|
399
399
|
"""Context manager entry point.
|
400
400
|
|
401
401
|
Returns:
|
402
|
-
|
402
|
+
Retab: The client instance
|
403
403
|
"""
|
404
404
|
return self
|
405
405
|
|
@@ -414,15 +414,15 @@ class UiForm(BaseUiForm):
|
|
414
414
|
self.close()
|
415
415
|
|
416
416
|
|
417
|
-
class
|
418
|
-
"""Asynchronous client for interacting with the
|
417
|
+
class AsyncRetab(BaseRetab):
|
418
|
+
"""Asynchronous client for interacting with the Retab API.
|
419
419
|
|
420
|
-
This client provides asynchronous access to all
|
420
|
+
This client provides asynchronous access to all Retab API resources including files, fine-tuning,
|
421
421
|
prompt optimization, documents, models, datasets, and schemas.
|
422
422
|
|
423
423
|
Args:
|
424
|
-
api_key (str, optional):
|
425
|
-
base_url (str, optional): Base URL for API requests. Defaults to https://api.
|
424
|
+
api_key (str, optional): Retab API key. If not provided, will look for RETAB_API_KEY env variable.
|
425
|
+
base_url (str, optional): Base URL for API requests. Defaults to https://api.retab.dev
|
426
426
|
timeout (float): Request timeout in seconds. Defaults to 240.0
|
427
427
|
max_retries (int): Maximum number of retries for failed requests. Defaults to 3
|
428
428
|
openai_api_key (str, optional): OpenAI API key. Will look for OPENAI_API_KEY env variable if not provided
|
@@ -683,11 +683,11 @@ class AsyncUiForm(BaseUiForm):
|
|
683
683
|
"""Closes the async HTTP client session."""
|
684
684
|
await self.client.aclose()
|
685
685
|
|
686
|
-
async def __aenter__(self) -> "
|
686
|
+
async def __aenter__(self) -> "AsyncRetab":
|
687
687
|
"""Async context manager entry point.
|
688
688
|
|
689
689
|
Returns:
|
690
|
-
|
690
|
+
AsyncRetab: The async client instance
|
691
691
|
"""
|
692
692
|
return self
|
693
693
|
|
@@ -52,7 +52,7 @@ class Consensus(SyncAPIResource, BaseConsensusMixin):
|
|
52
52
|
Dict containing the consensus dictionary and consensus likelihoods
|
53
53
|
|
54
54
|
Raises:
|
55
|
-
|
55
|
+
RetabAPIError: If the API request fails
|
56
56
|
"""
|
57
57
|
request = self._prepare_reconcile(list_dicts, reference_schema, mode, idempotency_key)
|
58
58
|
response = self._client._prepared_request(request)
|
@@ -87,7 +87,7 @@ class AsyncConsensus(AsyncAPIResource, BaseConsensusMixin):
|
|
87
87
|
Dict containing the consensus dictionary and consensus likelihoods
|
88
88
|
|
89
89
|
Raises:
|
90
|
-
|
90
|
+
RetabAPIError: If the API request fails
|
91
91
|
"""
|
92
92
|
request = self._prepare_reconcile(list_dicts, reference_schema, mode, idempotency_key)
|
93
93
|
response = await self._client._prepared_request(request)
|