retab 0.0.35__py3-none-any.whl → 0.0.37__py3-none-any.whl

{uiform → retab}/_utils/ai_models.py

@@ -61,7 +61,7 @@ def assert_valid_model_batch_processing(model: str) -> None:
             raise ValueError(f"Invalid base model in fine-tuned model '{model}'. Base model must be one of: {get_args(OpenAIModel)}")
         if not model_id or not model_id.strip():
             raise ValueError(f"Model ID cannot be empty in fine-tuned model '{model}'")
-    except ValueError as e:
+    except ValueError:
         if ":" not in model:
             raise ValueError(
                 f"Invalid model format: {model}. Must be either:\n"
@@ -90,7 +90,7 @@ def assert_valid_model_schema_generation(model: str) -> None:
             raise ValueError(f"Invalid base model in fine-tuned model '{model}'. Base model must be one of: {get_args(OpenAIModel)}")
         if not model_id or not model_id.strip():
             raise ValueError(f"Model ID cannot be empty in fine-tuned model '{model}'")
-    except ValueError as e:
+    except ValueError:
         if ":" not in model:
             raise ValueError(
                 f"Invalid model format: {model}. Must be either:\n"

{uiform → retab}/_utils/benchmarking.py

@@ -5,7 +5,7 @@ import shutil
 # The goal is to leverage this piece of code to open a jsonl file and get an analysis of the performance of the model using a one-liner.
 ############# BENCHMARKING MODELS #############
 from itertools import zip_longest
-from typing import Any, Callable, Literal, Optional
+from typing import Any, Callable, Literal, Optional, cast
 
 import pandas as pd  # type: ignore
 from Levenshtein import distance as levenshtein_distance
@@ -27,7 +27,7 @@ def normalize_string(text: str) -> str:
     if not text:
         return ""
     # Remove all non-alphanumeric characters and convert to lowercase
-    return re.sub(r'[^a-zA-Z0-9]', '', text).lower()
+    return re.sub(r"[^a-zA-Z0-9]", "", text).lower()
 
 
 def hamming_distance_padded(s: str, t: str) -> int:
@@ -45,7 +45,7 @@ def hamming_distance_padded(s: str, t: str) -> int:
     s = normalize_string(s)
     t = normalize_string(t)
 
-    return sum(a != b for a, b in zip_longest(s, t, fillvalue=' '))
+    return sum(a != b for a, b in zip_longest(s, t, fillvalue=" "))
 
 
 def hamming_similarity(str_1: str, str_2: str) -> float:
@@ -385,7 +385,7 @@ class EvalMetrics(BaseModel):
     distances: dict[dictionary_metrics, EvalMetric]
 
 
-def flatten_dict(d: dict[str, Any], parent_key: str = '', sep: str = '.') -> dict[str, Any]:
+def flatten_dict(d: dict[str, Any], parent_key: str = "", sep: str = ".") -> dict[str, Any]:
     """Flatten a nested dictionary with dot-separated keys."""
     items: list[tuple[str, Any]] = []
     for k, v in d.items():
@@ -408,16 +408,14 @@ def plot_metrics_with_uncertainty(analysis: dict[str, Any], uncertainties: Optio
     """
     # Flatten the dictionaries
     flattened_analysis = flatten_dict(analysis)
-    if uncertainties:
-        flattened_uncertainties = flatten_dict(uncertainties)
-    else:
-        uncertainties_list = None
-
     # Prepare data by matching fields
     fields = list(flattened_analysis.keys())
     similarities = [flattened_analysis[field] for field in fields]
 
+    # Prepare uncertainties if provided
+    uncertainties_list = None
     if uncertainties:
+        flattened_uncertainties = flatten_dict(uncertainties)
         uncertainties_list = [flattened_uncertainties.get(field, None) for field in fields]
 
     # Create a DataFrame
@@ -454,10 +452,11 @@ def plot_metrics_with_uncertainty(analysis: dict[str, Any], uncertainties: Optio
 
         if similarity is None:
             continue  # Skip fields with no similarity value
-
+        similarity = cast(float, similarity)
         # Calculate bar length and uncertainty range
         bar_len = round(similarity * scale)
         if uncertainty is not None and uncertainty > 0:
+            uncertainty = cast(float, uncertainty)
             uncertainty_start = max(0, round((similarity - uncertainty) * scale))
             uncertainty_end = min(bar_width, round((similarity + uncertainty) * scale))
         else:
@@ -465,21 +464,21 @@ def plot_metrics_with_uncertainty(analysis: dict[str, Any], uncertainties: Optio
             uncertainty_end = bar_len  # No uncertainty to display
 
         # Build the bar string
-        bar_string = ''
+        bar_string = ""
         for i in range(bar_width):
             if i < bar_len:
                 if i < uncertainty_start:
-                    char = '█'  # Solid block for certain part
+                    char = "█"  # Solid block for certain part
                 else:
-                    char = '█'  # Lighter block for uncertainty overlap
+                    char = "█"  # Lighter block for uncertainty overlap
             else:
                 if i < uncertainty_end:
-                    char = '░'  # Dash for upper uncertainty range
+                    char = "░"  # Dash for upper uncertainty range
                 else:
-                    char = ' '  # Space for empty area
+                    char = " "  # Space for empty area
             bar_string += char
 
         # Print the label and bar
-        score_field = f'[{similarity:.4f}]'
+        score_field = f"[{similarity:.4f}]"
 
         print(f"{field:<{label_width}} {score_field} | {bar_string}")

{uiform → retab}/_utils/chat.py

@@ -1,22 +1,17 @@
 import base64
-import io
 import logging
 from typing import List, Literal, Optional, Union, cast
 
 import requests
-from anthropic.types.content_block import ContentBlock
-from anthropic.types.image_block_param import ImageBlockParam, Source
+from anthropic.types.image_block_param import ImageBlockParam
 from anthropic.types.message_param import MessageParam
 from anthropic.types.text_block_param import TextBlockParam
-from anthropic.types.tool_result_block_param import ToolResultBlockParam
-from anthropic.types.tool_use_block_param import ToolUseBlockParam
 from google.genai.types import BlobDict, ContentDict, ContentUnionDict, PartDict  # type: ignore
 from openai.types.chat.chat_completion_content_part_image_param import ChatCompletionContentPartImageParam
 from openai.types.chat.chat_completion_content_part_input_audio_param import ChatCompletionContentPartInputAudioParam
 from openai.types.chat.chat_completion_content_part_param import ChatCompletionContentPartParam
 from openai.types.chat.chat_completion_content_part_text_param import ChatCompletionContentPartTextParam
 from openai.types.chat.chat_completion_message_param import ChatCompletionMessageParam
-from PIL import Image
 
 from ..types.chat import ChatCompletionUiformMessage
 
@@ -64,7 +59,7 @@ def convert_to_google_genai_format(messages: List[ChatCompletionUiformMessage])
             continue
         parts: list[PartDict] = []
 
-        message_content = message['content']
+        message_content = message["content"]
         if isinstance(message_content, str):
             # Direct string content is treated as the prompt for the SDK
             parts.append(PartDict(text=message_content))
@@ -74,8 +69,8 @@ def convert_to_google_genai_format(messages: List[ChatCompletionUiformMessage])
                 if part["type"] == "text":
                     parts.append(PartDict(text=part["text"]))
                 elif part["type"] == "image_url":
-                    url = part['image_url'].get('url', '')  # type: ignore
-                    if url.startswith('data:image'):
+                    url = part["image_url"].get("url", "")  # type: ignore
+                    if url.startswith("data:image"):
                         # Extract base64 data and add it to the formatted inputs
                         media_type, data_content = url.split(";base64,")
                         media_type = media_type.split("data:")[-1]  # => "image/jpeg"
@@ -133,24 +128,24 @@ def convert_to_anthropic_format(messages: List[ChatCompletionUiformMessage]) ->
         # -----------------------
         # Handle non-system roles
         # -----------------------
-        if isinstance(message['content'], str):
+        if isinstance(message["content"], str):
             # Direct string content is treated as a single text block
             content_blocks.append(
                 {
                     "type": "text",
-                    "text": message['content'],
+                    "text": message["content"],
                 }
             )
 
-        elif isinstance(message['content'], list):
+        elif isinstance(message["content"], list):
             # Handle structured content
-            for part in message['content']:
+            for part in message["content"]:
                 if part["type"] == "text":
                     part = cast(ChatCompletionContentPartTextParam, part)
                     content_blocks.append(
                         {
                             "type": "text",
-                            "text": part['text'],  # type: ignore
+                            "text": part["text"],  # type: ignore
                         }
                     )
 

{uiform → retab}/_utils/display.py

@@ -105,9 +105,6 @@ def count_image_tokens(image_url: str, detail: Literal["low", "high", "auto"] =
         total_tiles = tiles_wide * tiles_high
 
         return base_token_cost + (token_per_tile * total_tiles)
-    
-
-
 
 
 def process_jsonl_file(jsonl_path: str) -> List[TokenCounts]:

{uiform → retab}/_utils/json_schema.py

@@ -14,8 +14,8 @@ from email_validator import validate_email
 from pydantic import BaseModel, BeforeValidator, Field, create_model
 from pydantic.config import ConfigDict
 
-from uiform._utils.mime import generate_blake2b_hash_from_string
-from uiform.types.schemas.layout import Column, FieldItem, Layout, RefObject, Row, RowList
+from ..types.schemas.layout import Column, FieldItem, Layout, RefObject, Row, RowList
+from .mime import generate_blake2b_hash_from_string
 
 # **** Validation Functions ****
 
@@ -116,7 +116,7 @@ def validate_vat_number(v: Any) -> Optional[str]:
     try:
         if stdnum.eu.vat.is_valid(v_str):
             return stdnum.eu.vat.validate(v_str)
-    except:
+    except Exception:
         pass
     return None
 
@@ -150,7 +150,7 @@ def validate_email_address(v: Any) -> Optional[str]:
         return None
     try:
         return validate_email(v_str).normalized
-    except:
+    except Exception:
         return None
 
 
@@ -170,7 +170,7 @@ def validate_frenchpostcode(v: Any) -> Optional[str]:
         if not v_str.isdigit():
             return None
         return v_str
-    except:
+    except Exception:
         return None
 
 
@@ -201,7 +201,7 @@ def validate_un_code(v: Any) -> Optional[int]:
         val = int(float(v_str))  # handle numeric strings
         if 0 <= val <= 3481:
             return val
-    except:
+    except Exception:
         pass
     return None
 
@@ -242,7 +242,7 @@ def validate_integer(v: Any) -> Optional[int]:
         return None
     try:
         return int(float(v_str))
-    except:
+    except Exception:
         return None
 
 
@@ -257,7 +257,7 @@ def validate_float(v: Any) -> Optional[float]:
         return None
     try:
         return float(v_str)
-    except:
+    except Exception:
         return None
 
 
@@ -333,7 +333,7 @@ def validate_bool(v: Any) -> bool:
             return True
         elif v_str in false_values:
             return False
-    except:
+    except Exception:
         pass
 
     return False
@@ -2091,11 +2091,6 @@ def sanitize(instance: Any, schema: dict[str, Any]) -> Any:
     return __sanitize_instance(instance, expanded_schema)
 
 
-import copy
-import json
-from .mime import generate_blake2b_hash_from_string
-
-
 def compute_schema_data_id(json_schema: dict[str, Any]) -> str:
     """Returns the schema_data_id for a given JSON schema.
 

{uiform → retab}/_utils/mime.py

@@ -4,16 +4,17 @@ import io
 import json
 import mimetypes
 from pathlib import Path
-from typing import Literal, Sequence, TypeVar, get_args
+from typing import Sequence, TypeVar, get_args
 
 import httpx
 import PIL.Image
+import puremagic
 from pydantic import HttpUrl
 
 from ..types.mime import MIMEData
 from ..types.modalities import SUPPORTED_TYPES
 
-T = TypeVar('T')
+T = TypeVar("T")
 
 
 def generate_blake2b_hash_from_bytes(bytes_: bytes) -> str:
@@ -25,7 +26,7 @@ def generate_blake2b_hash_from_base64(base64_string: str) -> str:
 
 
 def generate_blake2b_hash_from_string(input_string: str) -> str:
-    return generate_blake2b_hash_from_bytes(input_string.encode('utf-8'))
+    return generate_blake2b_hash_from_bytes(input_string.encode("utf-8"))
 
 
 def generate_blake2b_hash_from_dict(input_dict: dict) -> str:
@@ -43,7 +44,7 @@ def convert_pil_image_to_mime_data(image: PIL.Image.Image) -> MIMEData:
     """
     # Convert PIL image to base64 string
     buffered = io.BytesIO()
-    choosen_format = image.format if (image.format and image.format.lower() in ['png', 'jpeg', 'gif', 'webp']) else "JPEG"
+    choosen_format = image.format if (image.format and image.format.lower() in ["png", "jpeg", "gif", "webp"]) else "JPEG"
     image.save(buffered, format=choosen_format)
     base64_content = base64.b64encode(buffered.getvalue()).decode("utf-8")
 
@@ -98,13 +99,11 @@ def prepare_mime_document(document: Path | str | bytes | io.IOBase | MIMEData |
     if isinstance(document, bytes):
         # `document` is already the raw bytes
         try:
-            import puremagic
-
             extension = puremagic.from_string(document)
             if extension.lower() in [".jpg", ".jpeg", ".jfif"]:
                 extension = ".jpeg"
-        except:
-            extension = '.txt'
+        except Exception:
+            extension = ".txt"
         file_bytes = document
         filename = "uploaded_file" + extension
     elif isinstance(document, io.IOBase):
@@ -112,19 +111,17 @@ def prepare_mime_document(document: Path | str | bytes | io.IOBase | MIMEData |
         file_bytes = document.read()
         filename = getattr(document, "name", "uploaded_file")
         filename = Path(filename).name
-    elif hasattr(document, 'unicode_string') and callable(getattr(document, 'unicode_string')):
+    elif hasattr(document, "unicode_string") and callable(getattr(document, "unicode_string")):
         with httpx.Client() as client:
             url: str = document.unicode_string()  # type: ignore
             response = client.get(url)
             response.raise_for_status()
             try:
-                import puremagic
-
                 extension = puremagic.from_string(response.content)
                 if extension.lower() in [".jpg", ".jpeg", ".jfif"]:
                     extension = ".jpeg"
-            except:
-                extension = '.txt'
+            except Exception:
+                extension = ".txt"
             file_bytes = response.content  # Fix: Use response.content instead of document
             filename = "uploaded_file" + extension
     else:
@@ -139,7 +136,7 @@ def prepare_mime_document(document: Path | str | bytes | io.IOBase | MIMEData |
     encoded_content = base64.b64encode(file_bytes).decode("utf-8")
     # Compute SHA-256 hash over the *base64-encoded* content
     hash_obj = hashlib.sha256(encoded_content.encode("utf-8"))
-    content_hash = hash_obj.hexdigest()
+    hash_obj.hexdigest()
 
     # Guess MIME type based on file extension
     guessed_type, _ = mimetypes.guess_type(filename)

{uiform → retab}/_utils/responses.py

@@ -77,13 +77,19 @@ def convert_from_openai_format(messages: list[ResponseInputItemParam]) -> list[C
     formatted_messages: list[ChatCompletionUiformMessage] = []
 
     for message in messages:
+        if "role" not in message or "content" not in message:
+            # Mandatory fields for a message
+            if message.get("type") != "message":
+                print(f"Not supported message type: {message.get('type')}... Skipping...")
+            continue
+
+        role = message["role"]
+        content = message["content"]
+
         if "type" not in message:
             # The type is required by all other sub-types of ResponseInputItemParam except for EasyInputMessageParam and Message, which are messages.
             message["type"] = "message"
 
-        if message["type"] != "message":
-            print(f"Not supported message type: {message['type']}... Skipping...")
-            continue
         role = message["role"]
         content = message["content"]
         formatted_content: str | list[ChatCompletionContentPartParam]

{uiform → retab}/_utils/stream_context_managers.py

@@ -1,7 +1,7 @@
 from contextlib import AbstractAsyncContextManager, AbstractContextManager
 from typing import Any, AsyncGenerator, Callable, Generator, TypeVar, Union
 
-T = TypeVar('T')
+T = TypeVar("T")
 
 
 class AsyncGeneratorContextManager(AbstractAsyncContextManager[AsyncGenerator[T, None]]):

{uiform → retab}/_utils/usage/usage.py

@@ -1,4 +1,4 @@
-from typing import Optional, Dict
+from typing import Optional
 
 from openai.types.completion_usage import CompletionUsage
 from pydantic import BaseModel, Field
@@ -71,14 +71,12 @@ def compute_api_call_cost(pricing: Pricing, usage: CompletionUsage, is_ft: bool
     total_cost = (total_text_cost + total_audio_cost) / 1e6
 
     # Apply fine-tuning price hike if applicable
-    if is_ft and hasattr(pricing, 'ft_price_hike'):
+    if is_ft and hasattr(pricing, "ft_price_hike"):
         total_cost *= pricing.ft_price_hike
 
     return Amount(value=total_cost, currency="USD")
 
 
-
-
 def compute_cost_from_model(model: str, usage: CompletionUsage) -> Amount:
     # Extract base model name for fine-tuned models like "ft:gpt-4o:uiform:4389573"
     is_ft = False
@@ -93,7 +91,7 @@ def compute_cost_from_model(model: str, usage: CompletionUsage) -> Amount:
     try:
         model_card = get_model_card(model)
         pricing = model_card.pricing
-    except ValueError as e:
+    except ValueError:
         raise ValueError(f"No pricing information found for model: {model}")
 
     return compute_api_call_cost(pricing, usage, is_ft)
@@ -124,46 +122,48 @@ class CompletionsUsage(BaseModel):
     model: Optional[str] = Field(default=None, description="When group_by=model, this field provides the model name of the grouped usage result.")
     batch: Optional[bool] = Field(default=None, description="When group_by=batch, this field tells whether the grouped usage result is batch or not.")
 
+
 ########################
 # DETAILED COST BREAKDOWN
 ########################
 
+
 class TokenCounts(BaseModel):
     """Detailed breakdown of token counts by type and category."""
-    
+
     # Prompt token counts
     prompt_regular_text: int
     prompt_cached_text: int
     prompt_audio: int
-    
+
     # Completion token counts
     completion_regular_text: int
     completion_audio: int
-    
+
     # Total tokens (should match sum of all components)
     total_tokens: int
 
 
 class CostBreakdown(BaseModel):
     """Detailed breakdown of API call costs by token type and usage category."""
-    
+
     # Total cost amount
     total: Amount
-    
+
     # Text token costs broken down by category
     text_prompt_cost: Amount
     text_cached_cost: Amount
     text_completion_cost: Amount
     text_total_cost: Amount
-    
+
     # Audio token costs broken down by category (if applicable)
     audio_prompt_cost: Optional[Amount] = None
     audio_completion_cost: Optional[Amount] = None
     audio_total_cost: Optional[Amount] = None
-    
+
     # Token counts for reference
     token_counts: TokenCounts
-    
+
     # Model and fine-tuning information
     model: str
     is_fine_tuned: bool = False
@@ -172,7 +172,7 @@ class CostBreakdown(BaseModel):
 def compute_api_call_cost_with_breakdown(pricing: Pricing, usage: CompletionUsage, model: str, is_ft: bool = False) -> CostBreakdown:
     """
     Computes a detailed price breakdown for the given token usage, based on the pricing.
-    
+
     Returns a CostBreakdown object containing costs broken down by token type and category.
     """
     # ----- Process prompt tokens -----
@@ -211,7 +211,7 @@ def compute_api_call_cost_with_breakdown(pricing: Pricing, usage: CompletionUsag
     cost_audio_prompt = 0.0
     cost_audio_completion = 0.0
     total_audio_cost = 0.0
-    
+
     if pricing.audio and (prompt_audio > 0 or completion_audio > 0):
         cost_audio_prompt = prompt_audio * pricing.audio.prompt
         cost_audio_completion = completion_audio * pricing.audio.completion
@@ -219,27 +219,27 @@ def compute_api_call_cost_with_breakdown(pricing: Pricing, usage: CompletionUsag
 
     # Convert to dollars (divide by 1M) and create Amount objects
     ft_multiplier = pricing.ft_price_hike if is_ft else 1.0
-    
+
     # Create Amount objects for each cost category
     text_prompt_amount = Amount(value=(cost_text_prompt / 1e6) * ft_multiplier, currency="USD")
     text_cached_amount = Amount(value=(cost_text_cached / 1e6) * ft_multiplier, currency="USD")
     text_completion_amount = Amount(value=(cost_text_completion / 1e6) * ft_multiplier, currency="USD")
     text_total_amount = Amount(value=(total_text_cost / 1e6) * ft_multiplier, currency="USD")
-    
+
     # Audio amounts (if applicable)
     audio_prompt_amount = None
     audio_completion_amount = None
     audio_total_amount = None
-    
+
     if pricing.audio and (prompt_audio > 0 or completion_audio > 0):
         audio_prompt_amount = Amount(value=(cost_audio_prompt / 1e6) * ft_multiplier, currency="USD")
         audio_completion_amount = Amount(value=(cost_audio_completion / 1e6) * ft_multiplier, currency="USD")
         audio_total_amount = Amount(value=(total_audio_cost / 1e6) * ft_multiplier, currency="USD")
-    
+
     # Total cost
     total_cost = (total_text_cost + total_audio_cost) / 1e6 * ft_multiplier
     total_amount = Amount(value=total_cost, currency="USD")
-    
+
     # Create TokenCounts object with token usage breakdown
     token_counts = TokenCounts(
         prompt_regular_text=prompt_regular_text,
@@ -247,9 +247,9 @@ def compute_api_call_cost_with_breakdown(pricing: Pricing, usage: CompletionUsag
         prompt_audio=prompt_audio,
         completion_regular_text=completion_regular_text,
         completion_audio=completion_audio,
-        total_tokens=usage.total_tokens
+        total_tokens=usage.total_tokens,
     )
-    
+
     return CostBreakdown(
         total=total_amount,
         text_prompt_cost=text_prompt_amount,
@@ -261,28 +261,28 @@ def compute_api_call_cost_with_breakdown(pricing: Pricing, usage: CompletionUsag
         audio_total_cost=audio_total_amount,
         token_counts=token_counts,
         model=model,
-        is_fine_tuned=is_ft
+        is_fine_tuned=is_ft,
     )
 
 
 def compute_cost_from_model_with_breakdown(model: str, usage: CompletionUsage) -> CostBreakdown:
     """
     Computes a detailed cost breakdown for an API call using the specified model and usage.
-    
+
     Args:
         model: The model name (can be a fine-tuned model like "ft:gpt-4o:uiform:4389573")
         usage: Token usage statistics for the API call
-        
+
     Returns:
         CostBreakdown object with detailed cost information
-        
+
     Raises:
         ValueError: If no pricing information is found for the model
     """
     # Extract base model name for fine-tuned models like "ft:gpt-4o:uiform:4389573"
     original_model = model
     is_ft = False
-    
+
     if model.startswith("ft:"):
         # Split by colon and take the second part (index 1) which contains the base model
         parts = model.split(":")
@@ -294,7 +294,7 @@ def compute_cost_from_model_with_breakdown(model: str, usage: CompletionUsage) -
     try:
         model_card = get_model_card(model)
         pricing = model_card.pricing
-    except ValueError as e:
+    except ValueError:
         raise ValueError(f"No pricing information found for model: {original_model}")
 
     return compute_api_call_cost_with_breakdown(pricing, usage, original_model, is_ft)