retab 0.0.36__py3-none-any.whl → 0.0.38__py3-none-any.whl

{uiform → retab}/resources/evals.py

@@ -1,32 +1,28 @@
-from typing import Any, Dict, List, Optional, TypedDict, Union, Literal
 from io import IOBase
 from pathlib import Path
+from typing import Any, Dict, List, Optional, TypedDict, Union
 
 import PIL.Image
+from openai.types.chat.chat_completion_reasoning_effort import ChatCompletionReasoningEffort
 from pydantic import HttpUrl
 
 from .._resource import AsyncAPIResource, SyncAPIResource
-from ..types.standards import PreparedRequest
+from .._utils.mime import prepare_mime_document
 from ..types.evals import (
+    CreateIterationRequest,
+    DistancesResult,
+    DocumentItem,
     Evaluation,
     EvaluationDocument,
     Iteration,
-    DistancesResult,
-    PredictionData,
-    AddIterationFromJsonlRequest,
-    DocumentItem,
     UpdateEvaluationDocumentRequest,
-    PredictionMetadata,
-    CreateIterationRequest,
+    UpdateEvaluationRequest,
 )
-from ..types.jobs.base import InferenceSettings
-
+from ..types.inference_settings import InferenceSettings
 from ..types.mime import MIMEData
-from .._utils.mime import prepare_mime_document
 from ..types.modalities import Modality
-from openai.types.chat.chat_completion_reasoning_effort import ChatCompletionReasoningEffort
-
-from tqdm import tqdm
+from ..types.browser_canvas import BrowserCanvas
+from ..types.standards import PreparedRequest
 
 
 class DeleteResponse(TypedDict):
@@ -82,21 +78,17 @@ class EvalsMixin:
         Only the provided fields will be updated. Fields set to None will be excluded from the update.
         """
         # Build a dictionary with only the provided fields
-        update_data = {}
-        if name is not None:
-            update_data["name"] = name
-        if project_id is not None:
-            update_data["project_id"] = project_id
-        if json_schema is not None:
-            update_data["json_schema"] = json_schema
-        if documents is not None:
-            update_data["documents"] = [doc.model_dump(exclude_none=True, mode="json") for doc in documents]
-        if iterations is not None:
-            update_data["iterations"] = [iter.model_dump(exclude_none=True, mode="json") for iter in iterations]
-        if default_inference_settings is not None:
-            update_data["default_inference_settings"] = default_inference_settings.model_dump(exclude_none=True, mode="json")
-
-        return PreparedRequest(method="PATCH", url=f"/v1/evals/{evaluation_id}", data=update_data)
+
+        update_request = UpdateEvaluationRequest(
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            documents=documents,
+            iterations=iterations,
+            default_inference_settings=default_inference_settings,
+        )
+
+        return PreparedRequest(method="PATCH", url=f"/v1/evals/{evaluation_id}", data=update_request.model_dump(exclude_none=True, mode="json"))
 
     def prepare_list(self, project_id: Optional[str] = None) -> PreparedRequest:
         params = {}
@@ -114,9 +106,7 @@ class DocumentsMixin:
 
     def prepare_create(self, evaluation_id: str, document: MIMEData, annotation: Dict[str, Any]) -> PreparedRequest:
         # Serialize the MIMEData
-
         document_item = DocumentItem(mime_data=document, annotation=annotation, annotation_metadata=None)
-
         return PreparedRequest(method="POST", url=f"/v1/evals/{evaluation_id}/documents", data=document_item.model_dump(mode="json"))
 
     def prepare_list(self, evaluation_id: str, filename: Optional[str] = None) -> PreparedRequest:
@@ -127,7 +117,6 @@ class DocumentsMixin:
 
     def prepare_update(self, evaluation_id: str, document_id: str, annotation: Dict[str, Any]) -> PreparedRequest:
         update_request = UpdateEvaluationDocumentRequest(annotation=annotation, annotation_metadata=None)
-
         return PreparedRequest(method="PUT", url=f"/v1/evals/{evaluation_id}/documents/{document_id}", data=update_request.model_dump(mode="json", exclude_none=True))
 
     def prepare_delete(self, evaluation_id: str, document_id: str) -> PreparedRequest:
@@ -153,10 +142,10 @@ class IterationsMixin:
         modality: Modality = "native",
         reasoning_effort: ChatCompletionReasoningEffort = "medium",
         image_resolution_dpi: int = 96,
-        browser_canvas: Literal['A3', 'A4', 'A5'] = 'A4',
+        browser_canvas: BrowserCanvas = "A4",
         n_consensus: int = 1,
     ) -> PreparedRequest:
-        props = InferenceSettings(
+        inference_settings = InferenceSettings(
             model=model,
             temperature=temperature,
             modality=modality,
@@ -166,18 +155,30 @@ class IterationsMixin:
             n_consensus=n_consensus,
         )
 
-        perform_iteration_request = CreateIterationRequest(inference_settings=props, json_schema=json_schema)
+        request = CreateIterationRequest(inference_settings=inference_settings, json_schema=json_schema)
 
-        return PreparedRequest(method="POST", url=f"/v1/evals/{evaluation_id}/iterations/create", data=perform_iteration_request.model_dump(exclude_none=True, mode="json"))
+        return PreparedRequest(method="POST", url=f"/v1/evals/{evaluation_id}/iterations/create", data=request.model_dump(exclude_none=True, mode="json"))
 
     def prepare_update(
-        self, iteration_id: str, json_schema: Dict[str, Any], model: str, temperature: float = 0.0, image_resolution_dpi: int = 96, browser_canvas: Literal['A3', 'A4', 'A5'] = 'A4'
+        self,
+        iteration_id: str,
+        json_schema: Dict[str, Any],
+        model: str,
+        temperature: float = 0.0,
+        modality: Modality = "native",
+        reasoning_effort: ChatCompletionReasoningEffort = "medium",
+        image_resolution_dpi: int = 96,
+        browser_canvas: BrowserCanvas = "A4",
+        n_consensus: int = 1,
     ) -> PreparedRequest:
         inference_settings = InferenceSettings(
             model=model,
             temperature=temperature,
+            modality=modality,
+            reasoning_effort=reasoning_effort,
             image_resolution_dpi=image_resolution_dpi,
             browser_canvas=browser_canvas,
+            n_consensus=n_consensus,
         )
 
         iteration_data = Iteration(id=iteration_id, json_schema=json_schema, inference_settings=inference_settings, predictions=[])
@@ -261,7 +262,13 @@ class Evals(SyncAPIResource, EvalsMixin):
             HTTPException if the request fails
         """
         request = self.prepare_update(
-            evaluation_id=evaluation_id, name=name, project_id=project_id, json_schema=json_schema, documents=documents, iterations=iterations, default_inference_settings=default_inference_settings
+            evaluation_id=evaluation_id,
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            documents=documents,
+            iterations=iterations,
+            default_inference_settings=default_inference_settings,
         )
         response = self._client._prepared_request(request)
         return Evaluation(**response)
@@ -429,7 +436,7 @@ class Iterations(SyncAPIResource, IterationsMixin):
         json_schema: Optional[Dict[str, Any]] = None,
         reasoning_effort: ChatCompletionReasoningEffort = "medium",
         image_resolution_dpi: int = 96,
-        browser_canvas: Literal['A3', 'A4', 'A5'] = 'A4',
+        browser_canvas: BrowserCanvas = "A4",
         n_consensus: int = 1,
     ) -> Iteration:
         """
@@ -445,7 +452,7 @@ class Iterations(SyncAPIResource, IterationsMixin):
             image_resolution_dpi: The DPI of the image. Defaults to 96.
             browser_canvas: The canvas size of the browser. Must be one of:
                 - "A3" (11.7in x 16.54in)
-                - "A4" (8.27in x 11.7in) 
+                - "A4" (8.27in x 11.7in)
                 - "A5" (5.83in x 8.27in)
                 Defaults to "A4".
             n_consensus: Number of consensus iterations to perform
@@ -572,7 +579,13 @@ class AsyncEvals(AsyncAPIResource, EvalsMixin):
             HTTPException if the request fails
         """
         request = self.prepare_update(
-            evaluation_id=evaluation_id, name=name, project_id=project_id, json_schema=json_schema, documents=documents, iterations=iterations, default_inference_settings=default_inference_settings
+            evaluation_id=evaluation_id,
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            documents=documents,
+            iterations=iterations,
+            default_inference_settings=default_inference_settings,
         )
         response = await self._client._prepared_request(request)
         return Evaluation(**response)
@@ -739,7 +752,7 @@ class AsyncIterations(AsyncAPIResource, IterationsMixin):
         json_schema: Optional[Dict[str, Any]] = None,
         reasoning_effort: ChatCompletionReasoningEffort = "medium",
         image_resolution_dpi: int = 96,
-        browser_canvas: Literal['A3', 'A4', 'A5'] = 'A4',
+        browser_canvas: BrowserCanvas = "A4",
         n_consensus: int = 1,
     ) -> Iteration:
         """
@@ -755,7 +768,7 @@ class AsyncIterations(AsyncAPIResource, IterationsMixin):
             image_resolution_dpi: The DPI of the image. Defaults to 96.
             browser_canvas: The canvas size of the browser. Must be one of:
                 - "A3" (11.7in x 16.54in)
-                - "A4" (8.27in x 11.7in) 
+                - "A4" (8.27in x 11.7in)
                 - "A5" (5.83in x 8.27in)
                 Defaults to "A4".
             n_consensus: Number of consensus iterations to perform

retab/resources/evaluations/__init__.py

@@ -0,0 +1,3 @@
+from .client import AsyncEvaluations, Evaluations
+
+__all__ = ["Evaluations", "AsyncEvaluations"]

retab/resources/evaluations/client.py

@@ -0,0 +1,301 @@
+from typing import Any, Dict, List
+
+from ..._resource import AsyncAPIResource, SyncAPIResource
+from ...types.evaluations import Evaluation, PatchEvaluationRequest, ListEvaluationParams, CreateEvaluation
+from ...types.inference_settings import InferenceSettings
+from ...types.standards import PreparedRequest, DeleteResponse, FieldUnset
+from .documents import Documents, AsyncDocuments
+from .iterations import Iterations, AsyncIterations
+
+
+class EvaluationsMixin:
+    def prepare_create(
+        self,
+        name: str,
+        json_schema: dict[str, Any],
+        project_id: str = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> PreparedRequest:
+        # Use CreateEvaluation model
+        eval_data = CreateEvaluation(
+            name=name,
+            json_schema=json_schema,
+            project_id=project_id,
+            default_inference_settings=default_inference_settings,
+        )
+        return PreparedRequest(method="POST", url="/v1/evaluations", data=eval_data.model_dump(exclude_none=True, mode="json"))
+
+    def prepare_get(self, evaluation_id: str) -> PreparedRequest:
+        return PreparedRequest(method="GET", url=f"/v1/evaluations/{evaluation_id}")
+
+    def prepare_update(
+        self,
+        evaluation_id: str,
+        name: str = FieldUnset,
+        project_id: str = FieldUnset,
+        json_schema: dict[str, Any] = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> PreparedRequest:
+        """
+        Prepare a request to update an evaluation with partial updates.
+
+        Only the provided fields will be updated. Fields set to None will be excluded from the update.
+        """
+        # Build a dictionary with only the provided fields
+        data = PatchEvaluationRequest(
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            default_inference_settings=default_inference_settings,
+        ).model_dump(exclude_unset=True, mode="json")
+
+        return PreparedRequest(method="PATCH", url=f"/v1/evaluations/{evaluation_id}", data=data)
+
+    def prepare_list(self, project_id: str = FieldUnset) -> PreparedRequest:
+        """
+        Prepare a request to list evaluations.
+
+        Usage:
+        >>> client.evals.list(project_id="project_id")   # List all evaluations for a project
+        >>> client.evals.list()                          # List all evaluations (no project_id)
+
+        This does not work:
+        >>> client.evals.list(project_id=None)
+
+        Args:
+            project_id: The project ID to list evaluations for
+
+        Returns:
+            PreparedRequest: The prepared request
+        """
+        params = ListEvaluationParams(project_id=project_id).model_dump(exclude_unset=True, exclude_defaults=True, mode="json")
+        return PreparedRequest(method="GET", url="/v1/evaluations", params=params)
+
+    def prepare_delete(self, id: str) -> PreparedRequest:
+        return PreparedRequest(method="DELETE", url=f"/v1/evaluations/{id}")
+
+
+class Evaluations(SyncAPIResource, EvaluationsMixin):
+    """Evaluations API wrapper"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.documents = Documents(self._client)
+        self.iterations = Iterations(self._client)
+
+    def create(
+        self,
+        name: str,
+        json_schema: dict[str, Any],
+        project_id: str = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> Evaluation:
+        """
+        Create a new evaluation.
+
+        Args:
+            name: The name of the evaluation
+            json_schema: The JSON schema for the evaluation
+            project_id: The project ID to associate with the evaluation
+            documents: The documents to associate with the evaluation
+            default_inference_settings: The default inference settings to associate with the evaluation
+
+        Returns:
+            Evaluation: The created evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_create(name, json_schema, project_id, default_inference_settings=default_inference_settings)
+        response = self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    def get(self, evaluation_id: str) -> Evaluation:
+        """
+        Get an evaluation by ID.
+
+        Args:
+            evaluation_id: The ID of the evaluation to retrieve
+
+        Returns:
+            Evaluation: The evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_get(evaluation_id)
+        response = self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    def update(
+        self,
+        evaluation_id: str,
+        name: str = FieldUnset,
+        project_id: str = FieldUnset,
+        json_schema: dict[str, Any] = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> Evaluation:
+        """
+        Update an evaluation with partial updates.
+
+        Args:
+            evaluation_id: The ID of the evaluation to update
+            name: Optional new name for the evaluation
+            project_id: Optional new project ID
+            json_schema: Optional new JSON schema
+            documents: Optional list of documents to update
+            iterations: Optional list of iterations to update
+            default_inference_settings: Optional annotation properties
+
+        Returns:
+            Evaluation: The updated evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_update(
+            evaluation_id=evaluation_id,
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            default_inference_settings=default_inference_settings,
+        )
+        response = self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    def list(self, project_id: str = FieldUnset) -> List[Evaluation]:
+        """
+        List evaluations for a project.
+
+        Args:
+            project_id: The project ID to list evaluations for
+
+        Returns:
+            List[Evaluation]: List of evaluations
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_list(project_id)
+        response = self._client._prepared_request(request)
+        return [Evaluation(**item) for item in response.get("data", [])]
+
+    def delete(self, evaluation_id: str) -> DeleteResponse:
+        """
+        Delete an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation to delete
+
+        Returns:
+            DeleteResponse: The response containing success status and ID
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_delete(evaluation_id)
+        return self._client._prepared_request(request)
+
+
+class AsyncEvaluations(AsyncAPIResource, EvaluationsMixin):
+    """Async Evaluations API wrapper"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.documents = AsyncDocuments(self._client)
+        self.iterations = AsyncIterations(self._client)
+
+    async def create(self, name: str, json_schema: Dict[str, Any], project_id: str = FieldUnset) -> Evaluation:
+        """
+        Create a new evaluation.
+
+        Args:
+            name: The name of the evaluation
+            json_schema: The JSON schema for the evaluation
+            project_id: The project ID to associate with the evaluation
+
+        Returns:
+            Evaluation: The created evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_create(name, json_schema, project_id=project_id)
+        response = await self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    async def get(self, evaluation_id: str) -> Evaluation:
+        """
+        Get an evaluation by ID.
+
+        Args:
+            evaluation_id: The ID of the evaluation to retrieve
+
+        Returns:
+            Evaluation: The evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_get(evaluation_id)
+        response = await self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    async def update(
+        self,
+        evaluation_id: str,
+        name: str = FieldUnset,
+        project_id: str = FieldUnset,
+        json_schema: dict[str, Any] = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> Evaluation:
+        """
+        Update an evaluation with partial updates.
+
+        Args:
+            id: The ID of the evaluation to update
+            name: Optional new name for the evaluation
+            project_id: Optional new project ID
+            json_schema: Optional new JSON schema
+            documents: Optional list of documents to update
+            iterations: Optional list of iterations to update
+            default_inference_settings: Optional annotation properties
+
+        Returns:
+            Evaluation: The updated evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_update(
+            evaluation_id=evaluation_id,
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            default_inference_settings=default_inference_settings,
+        )
+        response = await self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    async def list(self, project_id: str = FieldUnset) -> List[Evaluation]:
+        """
+        List evaluations for a project.
+
+        Args:
+            project_id: The project ID to list evaluations for
+
+        Returns:
+            List[Evaluation]: List of evaluations
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_list(project_id)
+        response = await self._client._prepared_request(request)
+        return [Evaluation(**item) for item in response.get("data", [])]
+
+    async def delete(self, evaluation_id: str) -> DeleteResponse:
+        """
+        Delete an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation to delete
+
+        Returns:
+            DeleteResponse: The response containing success status and ID
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_delete(evaluation_id)
+        return await self._client._prepared_request(request)