retab 0.0.40__py3-none-any.whl → 0.0.42__py3-none-any.whl

retab/utils/_model_cards/auto.yaml

@@ -0,0 +1,43 @@
+- model: "auto-large"
+  pricing:
+    text:
+      prompt: 2.00
+      cached_discount: 0.25
+      completion: 8.00
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+  temperature_support: true
+
+- model: "auto-small"
+  pricing:
+    text:
+      prompt: 0.4
+      cached_discount: 0.25
+      completion: 1.6
+    audio:
+      prompt: 0.7
+      cached_discount: 0.175
+      completion: 1000
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+  temperature_support: true
+
+- model: "auto-micro"
+  pricing:
+    text:
+      prompt: 0.1
+      cached_discount: 0.25
+      completion: 0.4
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+  temperature_support: true

retab/utils/_model_cards/gemini.yaml

@@ -0,0 +1,117 @@
+# gemini-2.5-pro family
+- model: "gemini-2.5-pro"
+  pricing:
+    text:
+      prompt: 1.25
+      cached_discount: 0.25
+      completion: 10.00
+    audio: null
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true
+
+- model: "gemini-2.5-pro-exp-03-25"
+  inherits: "gemini-2.5-pro"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+- model: "gemini-2.5-pro-preview-06-05"
+  inherits: "gemini-2.5-pro"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+- model: "gemini-2.5-pro-preview-05-06"
+  inherits: "gemini-2.5-pro"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+- model: "gemini-2.5-pro-preview-03-25"
+  inherits: "gemini-2.5-pro"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+- model: "gemini-2.5-flash"
+  pricing:
+    text:
+      prompt: 0.30
+      completion: 2.50
+    audio:
+      prompt: 1.00
+      completion: 1000
+  capabilities:
+    modalities: ["text", "image", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true
+
+- model: "gemini-2.5-flash-preview-05-20"
+  pricing:
+    text:
+      prompt: 0.15
+      completion: 0.60
+    audio: null
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+
+- model: "gemini-2.5-flash-preview-04-17"
+  inherits: "gemini-2.5-flash-preview-05-20"
+
+# gemini-2.0-flash family
+- model: "gemini-2.0-flash"
+  pricing:
+    text:
+      prompt: 0.1
+      cached_discount: 0.25
+      completion: 0.40
+    audio:
+      prompt: 0.7
+      cached_discount: 0.25
+      completion: 1000
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+  temperature_support: true
+
+- model: "gemini-2.0-flash-lite"
+  pricing:
+    text:
+      prompt: 0.075
+      completion: 0.30
+    audio:
+      prompt: 0.075
+      completion: 1000
+  capabilities:
+    modalities: ["text", "image", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "structured_outputs"]
+  temperature_support: true
+
+- model: "gemini-2.5-flash-lite-preview-06-17"
+  pricing:
+    text:
+      prompt: 0.10
+      completion: 0.40
+    audio:
+      prompt: 0.50
+      completion: 0.40
+  capabilities:
+    modalities: ["text", "image", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "structured_outputs"]
+  temperature_support: true
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true

retab/utils/_model_cards/openai.yaml

@@ -0,0 +1,301 @@
+
+# Reasoning models
+# o1 family 
+- model: "o1"
+  pricing:
+    text:
+      prompt: 15.00
+      cached_discount: 0.5
+      completion: 60.00
+    audio: null
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch"]
+    features: ["streaming", "function_calling", "structured_outputs"]
+  temperature_support: false
+  reasoning_effort_support: true
+
+- model: "o1-2024-12-17"
+  inherits: "o1"
+
+# o3 family
+- model: "o3"
+  pricing:
+    text:
+      prompt: 2.0
+      cached_discount: 0.25
+      completion: 8.0
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch"]
+    features: ["streaming", "function_calling", "structured_outputs", "schema_generation"]
+  temperature_support: false
+  reasoning_effort_support: true
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: true
+
+- model: "o3-2025-04-16"
+  inherits: "o3"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+# o4-mini family
+- model: "o4-mini"
+  pricing:
+    text:
+      prompt: 1.10
+      cached_discount: 0.25
+      completion: 4.40
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch"]
+    features: ["streaming", "function_calling", "structured_outputs", "schema_generation"]
+  temperature_support: false
+  reasoning_effort_support: true
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: true
+
+- model: "o4-mini-2025-04-16"
+  inherits: "o4-mini"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+# Chat models
+# gpt-4.1 family
+- model: "gpt-4.1"
+  pricing:
+    text:
+      prompt: 2.00
+      cached_discount: 0.25
+      completion: 8.00
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch"]
+    features: ["streaming", "function_calling", "structured_outputs", "schema_generation"]
+  temperature_support: true
+  reasoning_effort_support: false
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true
+
+- model: "gpt-4.1-2025-04-14"
+  inherits: "gpt-4.1"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+- model: "gpt-4.1-mini"
+  pricing:
+    text:
+      prompt: 0.40
+      cached_discount: 0.25
+      completion: 1.60
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch"]
+    features: ["streaming", "function_calling", "structured_outputs", "schema_generation"]
+  temperature_support: true
+  reasoning_effort_support: false
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true
+
+- model: "gpt-4.1-mini-2025-04-14"
+  inherits: "gpt-4.1-mini"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+- model: "gpt-4.1-nano"
+  pricing:
+    text:
+      prompt: 0.10
+      cached_discount: 0.25
+      completion: 0.40
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch"]
+    features: ["streaming", "function_calling", "structured_outputs", "schema_generation"]
+  temperature_support: true
+  reasoning_effort_support: false
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true
+
+- model: "gpt-4.1-nano-2025-04-14"
+  inherits: "gpt-4.1-nano"
+  permissions:
+    show_in_free_picker: false
+    show_in_paid_picker: false
+
+# gpt-4o family
+- model: "chatgpt-4o-latest"
+  pricing:
+    text:
+      prompt: 2.50
+      cached_discount: 0.5
+      completion: 10.00
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch", "fine_tuning"]
+    features: ["streaming", "function_calling", "structured_outputs", "fine_tuning", "distillation", "predicted_outputs"]
+  temperature_support: true
+
+- model: "gpt-4o"
+  inherits: "chatgpt-4o-latest"
+
+- model: "gpt-4o-2024-08-06"
+  inherits: "chatgpt-4o-latest"
+
+- model: "gpt-4o-2024-11-20"
+  inherits: "chatgpt-4o-latest"
+
+- model: "gpt-4o-2024-05-13"
+  pricing:
+    text:
+      prompt: 5.00
+      cached_discount: 0.5
+      completion: 15.00
+    audio: null
+    ft_price_hike: 1.5
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch", "fine_tuning"]
+    features: ["streaming", "function_calling", "structured_outputs", "fine_tuning", "distillation", "predicted_outputs"]
+  temperature_support: true
+
+# gpt-4o-audio family
+- model: "gpt-4o-audio-preview-2024-12-17"
+  pricing:
+    text:
+      prompt: 2.50
+      cached_discount: 0.5
+      completion: 10.00
+    audio:
+      prompt: 40.00
+      cached_discount: 0.2
+      completion: 80.00
+  capabilities:
+    modalities: ["text", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling"]
+  temperature_support: true
+
+- model: "gpt-4o-audio-preview-2024-10-01"
+  pricing:
+    text:
+      prompt: 2.50
+      cached_discount: 0.5
+      completion: 10.00
+    audio:
+      prompt: 100.00
+      cached_discount: 0.2
+      completion: 200.00
+  capabilities:
+    modalities: ["text", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling"]
+  temperature_support: true
+
+- model: "gpt-4o-realtime-preview-2024-12-17"
+  pricing:
+    text:
+      prompt: 5.00
+      cached_discount: 0.5
+      completion: 20.00
+    audio:
+      prompt: 40.00
+      cached_discount: 0.2
+      completion: 80.00
+  capabilities:
+    modalities: ["text", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling"]
+  temperature_support: true
+
+- model: "gpt-4o-realtime-preview-2024-10-01"
+  pricing:
+    text:
+      prompt: 5.00
+      cached_discount: 0.5
+      completion: 20.00
+    audio:
+      prompt: 100.00
+      cached_discount: 0.2
+      completion: 200.00
+  capabilities:
+    modalities: ["text", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling"]
+  temperature_support: true
+
+# gpt-4o-mini family
+- model: "gpt-4o-mini"
+  pricing:
+    text:
+      prompt: 0.15
+      cached_discount: 0.5
+      completion: 0.60
+    audio: null
+    ft_price_hike: 2.0
+  capabilities:
+    modalities: ["text", "image"]
+    endpoints: ["chat_completions", "responses", "assistants", "batch", "fine_tuning"]
+    features: ["streaming", "function_calling", "structured_outputs", "fine_tuning"]
+  temperature_support: true
+
+- model: "gpt-4o-mini-2024-07-18"
+  inherits: "gpt-4o-mini"
+
+# gpt-4o-mini-audio family
+- model: "gpt-4o-mini-audio-preview-2024-12-17"
+  pricing:
+    text:
+      prompt: 0.15
+      cached_discount: 0.5
+      completion: 0.60
+    audio:
+      prompt: 10.00
+      cached_discount: 0.2
+      completion: 20.00
+    ft_price_hike: 2.0
+  capabilities:
+    modalities: ["text", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling"]
+  temperature_support: true
+
+- model: "gpt-4o-mini-realtime-preview-2024-12-17"
+  pricing:
+    text:
+      prompt: 0.60
+      cached_discount: 0.5
+      completion: 2.40
+    audio:
+      prompt: 10.00
+      cached_discount: 0.2
+      completion: 20.00
+    ft_price_hike: 2.0
+  capabilities:
+    modalities: ["text", "audio"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "function_calling"]
+  temperature_support: true
+
+

retab/utils/_model_cards/xai.yaml

@@ -0,0 +1,28 @@
+# grok3-family
+- model: "grok-3"
+  pricing:
+    text:
+      prompt: 3
+      completion: 15
+    audio: null
+  capabilities:
+    modalities: ["text"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "structured_outputs"]
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true
+
+- model: "grok-3-mini"
+  pricing:
+    text:
+      prompt: 0.3
+      completion: 0.5
+    audio: null
+  capabilities:
+    modalities: ["text"]
+    endpoints: ["chat_completions"]
+    features: ["streaming", "structured_outputs"]
+  permissions:
+    show_in_free_picker: true
+    show_in_paid_picker: true

retab/utils/ai_models.py

@@ -0,0 +1,138 @@
+import os
+import yaml
+from typing import get_args
+
+from ..types.ai_models import AIProvider, GeminiModel, OpenAIModel, xAI_Model, RetabModel, PureLLMModel, ModelCard
+
+MODEL_CARDS_DIR = os.path.join(os.path.dirname(__file__), "_model_cards")
+
+def merge_model_cards(base: dict, override: dict) -> dict:
+    result = base.copy()
+    for key, value in override.items():
+        if key == "inherits":
+            continue
+        if isinstance(value, dict) and key in result:
+            result[key] = merge_model_cards(result[key], value)
+        else:
+            result[key] = value
+    return result
+
+def load_model_cards(yaml_file: str) -> list[ModelCard]:
+    raw_cards = yaml.safe_load(open(yaml_file))
+    name_to_card = {c["model"]: c for c in raw_cards if "inherits" not in c}
+
+    final_cards = []
+    for card in raw_cards:
+        if "inherits" in card:
+            parent = name_to_card[card["inherits"]]
+            merged = merge_model_cards(parent, card)
+            final_cards.append(ModelCard(**merged))
+        else:
+            final_cards.append(ModelCard(**card))
+    return final_cards
+
+# Load all model cards
+model_cards = sum([
+    load_model_cards(os.path.join(MODEL_CARDS_DIR, "openai.yaml")),
+    load_model_cards(os.path.join(MODEL_CARDS_DIR, "anthropic.yaml")),
+    load_model_cards(os.path.join(MODEL_CARDS_DIR, "xai.yaml")),
+    load_model_cards(os.path.join(MODEL_CARDS_DIR, "gemini.yaml")),
+    load_model_cards(os.path.join(MODEL_CARDS_DIR, "auto.yaml")),
+], [])
+model_cards_dict = {card.model: card for card in model_cards}
+
+
+# Validate that model cards
+all_model_names = set(model_cards_dict.keys())
+if all_model_names.symmetric_difference(set(get_args(PureLLMModel))):
+    raise ValueError(f"Mismatch between model cards and PureLLMModel type: {all_model_names.symmetric_difference(set(get_args(PureLLMModel)))}")
+
+
+def get_model_from_model_id(model_id: str) -> str:
+    """
+    Get the model name from the model id.
+    """
+    if model_id.startswith("ft:"):
+        parts = model_id.split(":")
+        return parts[1]
+    else:
+        return model_id
+
+
+def get_model_card(model: str) -> ModelCard:
+    """
+    Get the model card for a specific model.
+
+    Args:
+        model: The model name to look up
+
+    Returns:
+        The ModelCard for the specified model
+
+    Raises:
+        ValueError: If no model card is found for the specified model
+    """
+    model_name = get_model_from_model_id(model)
+    if model_name in model_cards_dict:
+        model_card = ModelCard(**model_cards_dict[model_name].model_dump())
+        if model_name != model:
+            # Fine-tuned model -> Change the name
+            model_card.model = model
+            # Remove the fine-tuning feature (if exists)
+            try:
+                model_card.capabilities.features.remove("fine_tuning")
+            except ValueError:
+                pass
+        return model_card
+
+    raise ValueError(f"No model card found for model: {model_name}")
+
+
+def get_provider_for_model(model_id: str) -> AIProvider:
+    """
+    Determine the AI provider associated with the given model identifier.
+    Returns one of: "Anthropic", "xAI", "OpenAI", "Gemini", "Retab" or None if unknown.
+    """
+    model_name = get_model_from_model_id(model_id)
+    # if model_name in get_args(AnthropicModel):
+    #     return "Anthropic"
+    # if model_name in get_args(xAI_Model):
+    #     return "xAI"
+    if model_name in get_args(OpenAIModel):
+        return "OpenAI"
+    if model_name in get_args(GeminiModel):
+        return "Gemini"
+    if model_name in get_args(RetabModel):
+        return "Retab"
+    raise ValueError(f"Unknown model: {model_name}")
+
+
+def assert_valid_model_extraction(model: str) -> None:
+    try: 
+        get_provider_for_model(model)
+    except ValueError:
+        raise ValueError(
+            f"Invalid model for extraction: {model}.\nValid OpenAI models: {get_args(OpenAIModel)}\n"
+            f"Valid xAI models: {get_args(xAI_Model)}\n"
+            f"Valid Gemini models: {get_args(GeminiModel)}"
+        ) from None
+
+
+def assert_valid_model_schema_generation(model: str) -> None:
+    """Assert that the model is either a standard OpenAI model or a valid fine-tuned model.
+
+    Valid formats:
+    - Standard model: Must be in OpenAIModel
+    - Fine-tuned model: Must be {base_model}:{id} where base_model is in OpenAIModel
+
+    Raises:
+        ValueError: If the model format is invalid
+    """
+    if get_model_from_model_id(model) in get_args(OpenAIModel):
+        return
+    else:
+        raise ValueError(
+                f"Invalid model format: {model}. Must be either:\n"
+                f"1. A standard model: {get_args(OpenAIModel)}\n"
+                f"2. A fine-tuned model in format 'base_model:id' where base_model is one of the standard openai models"
+            ) from None