retab 0.0.36__py3-none-any.whl → 0.0.37__py3-none-any.whl

{uiform → retab}/types/automations/outlook.py

@@ -1,10 +1,9 @@
 import re
-from typing import Any, Dict, List, Literal, Optional
+from typing import Any, Dict, List, Optional
 
 import nanoid  # type: ignore
-from pydantic import BaseModel, EmailStr, Field, field_validator, model_validator
+from pydantic import BaseModel, EmailStr, Field, computed_field, field_validator
 
-from ..._utils.json_schema import convert_schema_to_layout
 from ..logs import AutomationConfig, UpdateAutomationRequest
 from ..pagination import ListMetadata
 
@@ -29,7 +28,11 @@ class FetchParams(BaseModel):
 
 
 class Outlook(AutomationConfig):
-    object: Literal['automation.outlook'] = "automation.outlook"
+    @computed_field
+    @property
+    def object(self) -> str:
+        return "automation.outlook"
+
     id: str = Field(default_factory=lambda: "outlook_" + nanoid.generate(), description="Unique identifier for the outlook")
 
     authorized_domains: list[str] = Field(default_factory=list, description="List of authorized domains to receive the emails from")
@@ -41,12 +44,13 @@ class Outlook(AutomationConfig):
     match_params: List[MatchParams] = Field(default_factory=list, description="List of match parameters for the outlook automation")
     fetch_params: List[FetchParams] = Field(default_factory=list, description="List of fetch parameters for the outlook automation")
 
-    @model_validator(mode='before')
-    @classmethod
-    def compute_layout_schema(cls, values: dict[str, Any]) -> dict[str, Any]:
-        if values.get('layout_schema') is None:
-            values['layout_schema'] = convert_schema_to_layout(values['json_schema'])
-        return values
+    # @model_validator(mode="before")
+    # @classmethod
+    # def compute_layout_schema(cls, values: dict[str, Any]) -> dict[str, Any]:
+    #     if values.get("layout_schema") is None:
+    #         values["layout_schema"] = convert_schema_to_layout(values["json_schema"])
+    #     return values
+
 
 class ListOutlooks(BaseModel):
     data: list[Outlook]
@@ -65,4 +69,4 @@ class UpdateOutlookRequest(UpdateAutomationRequest):
 
     @field_validator("authorized_emails", mode="before")
     def normalize_authorized_emails(cls, emails: Optional[List[str]]) -> Optional[List[str]]:
-        return [email.strip().lower() for email in emails] if emails else None
+        return [email.strip().lower() for email in emails] if emails else None

retab/types/browser_canvas.py

@@ -0,0 +1,3 @@
+from typing import Literal
+
+BrowserCanvas = Literal["A3", "A4", "A5"]

{uiform → retab}/types/chat.py

@@ -1,8 +1,8 @@
-from typing import Iterable, Literal, TypedDict, Union
+from typing import Literal, TypedDict, Union
 
 from openai.types.chat.chat_completion_content_part_param import ChatCompletionContentPartParam
 
 
 class ChatCompletionUiformMessage(TypedDict):  # homemade replacement for ChatCompletionMessageParam because iterable messes the serialization with pydantic
-    role: Literal['user', 'system', 'assistant', 'developer']
+    role: Literal["user", "system", "assistant", "developer"]
     content: Union[str, list[ChatCompletionContentPartParam]]

{uiform → retab}/types/completions.py

@@ -1,6 +1,10 @@
-from typing import Any
+from typing import Any, Optional, Union
 
 from openai.types.chat.chat_completion_reasoning_effort import ChatCompletionReasoningEffort
+from openai.types.responses.response_input_param import ResponseInputParam
+from openai.types.responses.response_text_config_param import ResponseTextConfigParam
+from openai.types.shared_params.reasoning import Reasoning
+from openai.types.shared_params.response_format_json_schema import ResponseFormatJSONSchema
 from pydantic import BaseModel, ConfigDict, Field
 
 from .._utils.ai_models import find_provider_from_model
@@ -8,8 +12,6 @@ from .ai_models import AIProvider
 from .chat import ChatCompletionUiformMessage
 
 
-from openai.types.shared_params.response_format_json_schema import ResponseFormatJSONSchema
-
 class UiChatCompletionsRequest(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
     model: str = Field(..., description="Model used for chat completion")
@@ -35,8 +37,6 @@ class UiChatCompletionsRequest(BaseModel):
         return find_provider_from_model(self.model)
 
 
-
-
 class UiChatCompletionsParseRequest(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
     model: str = Field(..., description="Model used for chat completion")
@@ -61,10 +61,6 @@ class UiChatCompletionsParseRequest(BaseModel):
         """
         return find_provider_from_model(self.model)
 
-from typing import Optional, Union
-from openai.types.shared_params.reasoning import Reasoning
-from openai.types.responses.response_input_param import ResponseInputParam
-from openai.types.responses.response_text_config_param import ResponseTextConfigParam
 
 class UiChatResponseCreateRequest(BaseModel):
     input: Union[str, ResponseInputParam] = Field(..., description="Input to be parsed")
@@ -73,8 +69,10 @@ class UiChatResponseCreateRequest(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
     model: str = Field(..., description="Model used for chat completion")
     temperature: Optional[float] = Field(default=0.0, description="Temperature for sampling. If not provided, the default temperature for the model will be used.", examples=[0.0])
-    reasoning: Optional[Reasoning] = Field(default=None, description="The effort level for the model to reason about the input data. If not provided, the default reasoning effort for the model will be used.")
-    
+    reasoning: Optional[Reasoning] = Field(
+        default=None, description="The effort level for the model to reason about the input data. If not provided, the default reasoning effort for the model will be used."
+    )
+
     stream: Optional[bool] = Field(default=False, description="If true, the extraction will be streamed to the user using the active WebSocket connection")
     seed: int | None = Field(default=None, description="Seed for the random number generator. If not provided, a random seed will be generated.", examples=[None])
     text: ResponseTextConfigParam = Field(default={"format": {"type": "text"}}, description="Format of the response")
@@ -90,4 +88,3 @@ class UiChatResponseCreateRequest(BaseModel):
             AIProvider: The AI provider corresponding to the given model.
         """
         return find_provider_from_model(self.model)
-

retab/types/consensus.py

@@ -0,0 +1,19 @@
+from typing import Any, Literal, Optional
+from pydantic import BaseModel, Field
+
+
+class ReconciliationRequest(BaseModel):
+    list_dicts: list[dict] = Field(description="List of dictionaries that will be reconciled into a single consensus dictionary.")
+    reference_schema: Optional[dict[str, Any]] = Field(
+        default=None,
+        description="Optional schema defining the structure and types of the dictionary fields to validate the list of dictionaries against. Raise an error if one of the dictionaries does not match the schema.",
+    )
+    mode: Literal["direct", "aligned"] = Field(
+        default="direct",
+        description="The mode to use for the consensus. If 'direct', the consensus is computed directly from the list of dictionaries. If 'aligned', the consensus is computed from the aligned dictionaries.",
+    )
+
+
+class ReconciliationResponse(BaseModel):
+    consensus_dict: dict = Field(description="The consensus dictionary containing the reconciled values from the input dictionaries.")
+    likelihoods: dict = Field(description="A dictionary containing the likelihood/confidence scores for each field in the consensus dictionary.")

{uiform → retab}/types/db/annotations.py

@@ -1,17 +1,17 @@
 import datetime
-from typing import Any, Dict, Literal
+from typing import Any, Dict
 
-import nanoid  # type: ignore
 from pydantic import BaseModel, Field
 
 from ..modalities import Modality
+from ..browser_canvas import BrowserCanvas
 
 
 class AnnotationParameters(BaseModel):
     model: str
     modality: Modality | None = "native"
     image_resolution_dpi: int = 96
-    browser_canvas: Literal['A3', 'A4', 'A5'] = 'A4'
+    browser_canvas: BrowserCanvas = "A4"
     temperature: float = 0.0
 
 

{uiform → retab}/types/db/files.py

@@ -1,7 +1,7 @@
 import mimetypes
-from typing import BinaryIO, Literal, Tuple
+from typing import Any, BinaryIO, Literal, Tuple
 
-from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_serializer
+from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
 
 
 class DBFile(BaseModel):
@@ -27,10 +27,12 @@ FileTuple = Tuple[str, FileData]
 
 
 class FileLink(BaseModel):
-    download_url: HttpUrl = Field(description="The signed URL to download the file")
+    download_url: str = Field(description="The signed URL to download the file")
     expires_in: str = Field(description="The expiration time of the signed URL")
     filename: str = Field(description="The name of the file")
 
-    @field_serializer('download_url')
-    def url2str(self, val: HttpUrl) -> str:
-        return str(val)
+    @field_validator("download_url", mode="after")
+    def validate_httpurl(cls, val: Any) -> Any:
+        if isinstance(val, str):
+            HttpUrl(val)
+        return val

{uiform → retab}/types/documents/create_messages.py

@@ -1,6 +1,6 @@
 import base64
 from io import BytesIO
-from typing import List, Literal, Dict, Union
+from typing import Any, List, Literal
 
 import PIL.Image
 import requests
@@ -12,11 +12,12 @@ from pydantic import BaseModel, Field, computed_field
 
 from ..._utils.chat import convert_to_anthropic_format, convert_to_google_genai_format, str_messages
 from ..._utils.chat import convert_to_openai_format as convert_to_openai_completions_api_format
+from ..._utils.display import count_image_tokens, count_text_tokens
 from ..._utils.responses import convert_to_openai_format as convert_to_openai_responses_api_format
-from ..._utils.display import count_text_tokens, count_image_tokens
 from ..chat import ChatCompletionUiformMessage
 from ..mime import MIMEData
 from ..modalities import Modality
+from ..browser_canvas import BrowserCanvas
 
 MediaType = Literal["image/jpeg", "image/png", "image/gif", "image/webp"]
 
@@ -26,19 +27,20 @@ class TokenCount(BaseModel):
     developer_tokens: int = 0
     user_tokens: int = 0
 
+
 class DocumentCreateMessageRequest(BaseModel):
     document: MIMEData = Field(description="The document to load.")
     modality: Modality = Field(description="The modality of the document to load.")
     image_resolution_dpi: int = Field(default=96, description="Resolution of the image sent to the LLM")
-    browser_canvas: Literal['A3', 'A4', 'A5'] = Field(default='A4', description="Sets the size of the browser canvas for rendering documents in browser-based processing. Choose a size that matches the document type.")
+    browser_canvas: BrowserCanvas = Field(
+        default="A4", description="Sets the size of the browser canvas for rendering documents in browser-based processing. Choose a size that matches the document type."
+    )
+
 
-from typing import Any
 class DocumentCreateInputRequest(DocumentCreateMessageRequest):
     json_schema: dict[str, Any] = Field(description="The json schema to use for the document.")
 
 
-
-
 class DocumentMessage(BaseModel):
     id: str = Field(description="A unique identifier for the document loading.")
     object: Literal["document_message"] = Field(default="document_message", description="The type of object being loaded.")
@@ -49,21 +51,21 @@ class DocumentMessage(BaseModel):
     @computed_field
     def token_count(self) -> TokenCount:
         """Returns the token count for the document message.
-        
+
         This property calculates token usage based on both text and image content
         in the messages using the token counting utilities.
-        
+
         Returns:
             TokenCount: A Pydantic model with total, user, and developer token counts.
         """
         total_tokens = 0
         user_tokens = 0
         developer_tokens = 0
-        
+
         for msg in self.messages:
             role = msg.get("role", "user")
             msg_tokens = 0
-            
+
             if isinstance(msg["content"], str):
                 msg_tokens = count_text_tokens(msg["content"])
             elif isinstance(msg["content"], list):
@@ -72,30 +74,26 @@ class DocumentMessage(BaseModel):
                         msg_tokens += count_text_tokens(content_item)
                     elif isinstance(content_item, dict):
                         item_type = content_item.get("type")
-                        
+
                         if item_type == "text" and "text" in content_item:
                             msg_tokens += count_text_tokens(content_item["text"])
-                        
+
                         elif item_type == "image_url" and "image_url" in content_item:
                             image_url = content_item["image_url"]["url"]
                             detail = content_item["image_url"].get("detail", "high")
                             msg_tokens += count_image_tokens(image_url, detail)
-            
+
             # Update total tokens
             total_tokens += msg_tokens
-            
+
             # Update role-specific counts
             assert role in ["user", "developer"], f"Invalid role: {role}"
             if role == "user":
                 user_tokens += msg_tokens
             elif role == "developer":
                 developer_tokens += msg_tokens
-        
-        return TokenCount(
-            total_tokens=total_tokens,
-            user_tokens=user_tokens,
-            developer_tokens=developer_tokens
-        )
+
+        return TokenCount(total_tokens=total_tokens, user_tokens=user_tokens, developer_tokens=developer_tokens)
 
     @property
     def items(self) -> list[str | PIL.Image.Image]:

{uiform → retab}/types/documents/extractions.py

@@ -15,24 +15,23 @@ from openai.types.responses.response import Response
 from openai.types.responses.response_input_param import ResponseInputItemParam
 from pydantic import BaseModel, ConfigDict, Field, ValidationInfo, computed_field, field_validator, model_validator
 
-from ..._utils.usage.usage import compute_cost_from_model, compute_cost_from_model_with_breakdown, CostBreakdown
-
-from ..._utils.ai_models import find_provider_from_model
-from ..ai_models import AIProvider, Amount, get_model_card
+from ..._utils.usage.usage import CostBreakdown, compute_cost_from_model, compute_cost_from_model_with_breakdown
+from ..ai_models import Amount
 from ..chat import ChatCompletionUiformMessage
 from ..mime import MIMEData
 from ..modalities import Modality
+from ..browser_canvas import BrowserCanvas
 from ..standards import ErrorDetail, StreamingBaseModel
 
 
 class DocumentExtractRequest(BaseModel):
     model_config = ConfigDict(arbitrary_types_allowed=True)
-
-    document: MIMEData = Field(..., description="Document to be analyzed")
+    document: MIMEData = Field(default=None, description="Document to be analyzed", deprecated=True)  # type: ignore
+    documents: list[MIMEData] = Field(..., description="Documents to be analyzed (preferred over document)")
     modality: Modality
     image_resolution_dpi: int = Field(default=96, description="Resolution of the image sent to the LLM")
-    browser_canvas: Literal['A3', 'A4', 'A5'] = Field(
-        default='A4', description="Sets the size of the browser canvas for rendering documents in browser-based processing. Choose a size that matches the document type."
+    browser_canvas: BrowserCanvas = Field(
+        default="A4", description="Sets the size of the browser canvas for rendering documents in browser-based processing. Choose a size that matches the document type."
     )
     model: str = Field(..., description="Model used for chat completion")
     json_schema: dict[str, Any] = Field(..., description="JSON schema format used to validate the output data.")
@@ -54,6 +53,28 @@ class DocumentExtractRequest(BaseModel):
             raise ValueError("n_consensus greater than 1 but temperature is 0")
         return v
 
+    @model_validator(mode="before")
+    def validate_document_or_documents(cls, data: Any) -> Any:
+        # Handle both dict and model instance cases
+        if isinstance(data, dict):
+            if data.get("documents"):  # If documents is set, it has higher priority than document
+                data["document"] = data["documents"][0]
+            elif data.get("document"):
+                data["documents"] = [data["document"]]
+            else:
+                raise ValueError("document or documents must be provided")
+        else:
+            # Handle model instance case
+            document = getattr(data, "document", None)
+            documents = getattr(data, "documents", None)
+            if documents:
+                setattr(data, "document", documents[0])
+            elif document:
+                setattr(data, "documents", [document])
+            else:
+                raise ValueError("document or documents must be provided")
+        return data
+
 
 class ConsensusModel(BaseModel):
     model: str = Field(description="Model name")
@@ -87,7 +108,7 @@ LikelihoodsSource = Literal["consensus", "log_probs"]
 
 class UiParsedChatCompletion(ParsedChatCompletion):
     extraction_id: str | None = None
-    choices: list[UiParsedChoice]
+    choices: list[UiParsedChoice]  # type: ignore
     # Additional metadata fields (UIForm)
     likelihoods: Optional[dict[str, Any]] = Field(
         default=None, description="Object defining the uncertainties of the fields extracted when using consensus. Follows the same structure as the extraction object."
@@ -147,19 +168,43 @@ class LogExtractionRequest(BaseModel):
     # Validate that at least one of the messages, openai_messages, anthropic_messages is provided using model_validator
     @model_validator(mode="before")
     def validation(cls, data: Any) -> Any:
-        messages_candidates = [data.get("messages"), data.get("openai_messages"), data.get("anthropic_messages"), data.get("openai_responses_input")]
-        messages_candidates = [candidate for candidate in messages_candidates if candidate is not None]
-        if len(messages_candidates) != 1:
-            raise ValueError("Exactly one of the messages, openai_messages, anthropic_messages, openai_responses_input must be provided")
-
-        # Validate that if anthropic_messages is provided, anthropic_system_prompt is also provided
-        if data.get("anthropic_messages") is not None and data.get("anthropic_system_prompt") is None:
-            raise ValueError("anthropic_system_prompt must be provided if anthropic_messages is provided")
-
-        completion_candidates = [data.get("completion"), data.get("openai_responses_output")]
-        completion_candidates = [candidate for candidate in completion_candidates if candidate is not None]
-        if len(completion_candidates) != 1:
-            raise ValueError("Exactly one of completion, openai_responses_output must be provided")
+        # Handle both dict and model instance cases
+        if isinstance(data, dict):
+            messages_candidates = [data.get("messages"), data.get("openai_messages"), data.get("anthropic_messages"), data.get("openai_responses_input")]
+            messages_candidates = [candidate for candidate in messages_candidates if candidate is not None]
+            if len(messages_candidates) != 1:
+                raise ValueError("Exactly one of the messages, openai_messages, anthropic_messages, openai_responses_input must be provided")
+
+            # Validate that if anthropic_messages is provided, anthropic_system_prompt is also provided
+            if data.get("anthropic_messages") is not None and data.get("anthropic_system_prompt") is None:
+                raise ValueError("anthropic_system_prompt must be provided if anthropic_messages is provided")
+
+            completion_candidates = [data.get("completion"), data.get("openai_responses_output")]
+            completion_candidates = [candidate for candidate in completion_candidates if candidate is not None]
+            if len(completion_candidates) != 1:
+                raise ValueError("Exactly one of completion, openai_responses_output must be provided")
+        else:
+            # Handle model instance case
+            messages_candidates = [
+                getattr(data, "messages", None),
+                getattr(data, "openai_messages", None),
+                getattr(data, "anthropic_messages", None),
+                getattr(data, "openai_responses_input", None),
+            ]
+            messages_candidates = [candidate for candidate in messages_candidates if candidate is not None]
+            if len(messages_candidates) != 1:
+                raise ValueError("Exactly one of the messages, openai_messages, anthropic_messages, openai_responses_input must be provided")
+
+            # Validate that if anthropic_messages is provided, anthropic_system_prompt is also provided
+            anthropic_messages = getattr(data, "anthropic_messages", None)
+            anthropic_system_prompt = getattr(data, "anthropic_system_prompt", None)
+            if anthropic_messages is not None and anthropic_system_prompt is None:
+                raise ValueError("anthropic_system_prompt must be provided if anthropic_messages is provided")
+
+            completion_candidates = [getattr(data, "completion", None), getattr(data, "openai_responses_output", None)]
+            completion_candidates = [candidate for candidate in completion_candidates if candidate is not None]
+            if len(completion_candidates) != 1:
+                raise ValueError("Exactly one of completion, openai_responses_output must be provided")
 
         return data
 
@@ -192,12 +237,12 @@ class UiParsedChoiceDeltaChunk(ChoiceDeltaChunk):
 
 
 class UiParsedChoiceChunk(ChoiceChunk):
-    delta: UiParsedChoiceDeltaChunk
+    delta: UiParsedChoiceDeltaChunk  # type: ignore
 
 
 class UiParsedChatCompletionChunk(StreamingBaseModel, ChatCompletionChunk):
     extraction_id: str | None = None
-    choices: list[UiParsedChoiceChunk]
+    choices: list[UiParsedChoiceChunk]  # type: ignore
     schema_validation_error: ErrorDetail | None = None
     # Timestamps
     request_at: datetime.datetime | None = Field(default=None, description="Timestamp of the request")

{uiform → retab}/types/evals.py

@@ -1,19 +1,17 @@
 import copy
 import datetime
 import json
-from typing import Any, List, Literal, Optional, Union
+from typing import Any, List, Literal, Optional
 
 import nanoid  # type: ignore
 from pydantic import BaseModel, Field, computed_field
 
-
 from .._utils.json_schema import clean_schema, compute_schema_data_id
 from .._utils.mime import generate_blake2b_hash_from_string
-from .ai_models import Amount, LLMModel
-from .jobs.base import InferenceSettings
+from .ai_models import Amount
+from .inference_settings import InferenceSettings
 from .mime import MIMEData
 
-
 # Define the type alias for MetricType
 MetricType = Literal["levenshtein", "jaccard", "hamming"]
 
@@ -132,6 +130,7 @@ class UpdateEvaluationRequest(BaseModel):
     json_schema: Optional[dict[str, Any]] = Field(default=None, description="The json schema of the evaluation")
 
     project_id: Optional[str] = Field(default=None, description="The ID of the project")
+    default_inference_settings: Optional[InferenceSettings] = Field(default=None, description="The default inference properties for the evaluation (mostly used in the frontend)")
 
     @computed_field  # type: ignore
     @property
@@ -165,6 +164,7 @@ class Evaluation(BaseModel):
     updated_at: datetime.datetime = Field(default_factory=lambda: datetime.datetime.now(tz=datetime.timezone.utc))
 
     name: str
+    old_documents: list[EvaluationDocument] | None = None
     documents: list[EvaluationDocument]
     iterations: list[Iteration]
     json_schema: dict[str, Any]

retab/types/evaluations/__init__.py

@@ -0,0 +1,31 @@
+from .model import Evaluation, CreateEvaluation, PatchEvaluationRequest, ListEvaluationParams
+from .documents import AnnotatedDocument, DocumentItem, EvaluationDocument, CreateEvaluationDocumentRequest, PatchEvaluationDocumentRequest
+from .iterations import (
+    Iteration,
+    CreateIterationRequest,
+    PatchIterationRequest,
+    ProcessIterationRequest,
+    DocumentStatus,
+    IterationDocumentStatusResponse,
+    AddIterationFromJsonlRequest,
+)
+
+
+__all__ = [
+    "Evaluation",
+    "CreateEvaluation",
+    "PatchEvaluationRequest",
+    "ListEvaluationParams",
+    "AnnotatedDocument",
+    "DocumentItem",
+    "EvaluationDocument",
+    "CreateEvaluationDocumentRequest",
+    "PatchEvaluationDocumentRequest",
+    "Iteration",
+    "CreateIterationRequest",
+    "PatchIterationRequest",
+    "ProcessIterationRequest",
+    "DocumentStatus",
+    "IterationDocumentStatusResponse",
+    "AddIterationFromJsonlRequest",
+]

retab/types/evaluations/documents.py

@@ -0,0 +1,30 @@
+from typing import Any, Optional
+
+from pydantic import BaseModel, Field
+
+from ..mime import MIMEData
+from ..predictions import PredictionMetadata
+
+
+class AnnotatedDocument(BaseModel):
+    mime_data: MIMEData = Field(
+        description="The mime data of the document. Can also be a BaseMIMEData, which is why we have this id field (to be able to identify the file, but id is equal to mime_data.id)"
+    )
+    annotation: dict[str, Any] = Field(default={}, description="The ground truth of the document")
+
+
+class DocumentItem(AnnotatedDocument):
+    annotation_metadata: Optional[PredictionMetadata] = Field(default=None, description="The metadata of the annotation when the annotation is a prediction")
+
+
+class EvaluationDocument(DocumentItem):
+    id: str = Field(description="The ID of the document. Equal to mime_data.id but robust to the case where mime_data is a BaseMIMEData")
+
+
+class CreateEvaluationDocumentRequest(DocumentItem):
+    pass
+
+
+class PatchEvaluationDocumentRequest(BaseModel):
+    annotation: Optional[dict[str, Any]] = Field(default=None, description="The ground truth of the document")
+    annotation_metadata: Optional[PredictionMetadata] = Field(default=None, description="The metadata of the annotation when the annotation is a prediction")