retab 0.0.36__py3-none-any.whl → 0.0.37__py3-none-any.whl

retab/resources/evaluations/__init__.py

@@ -0,0 +1,3 @@
+from .client import AsyncEvaluations, Evaluations
+
+__all__ = ["Evaluations", "AsyncEvaluations"]

retab/resources/evaluations/client.py

@@ -0,0 +1,301 @@
+from typing import Any, Dict, List
+
+from ..._resource import AsyncAPIResource, SyncAPIResource
+from ...types.evaluations import Evaluation, PatchEvaluationRequest, ListEvaluationParams, CreateEvaluation
+from ...types.inference_settings import InferenceSettings
+from ...types.standards import PreparedRequest, DeleteResponse, FieldUnset
+from .documents import Documents, AsyncDocuments
+from .iterations import Iterations, AsyncIterations
+
+
+class EvaluationsMixin:
+    def prepare_create(
+        self,
+        name: str,
+        json_schema: dict[str, Any],
+        project_id: str = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> PreparedRequest:
+        # Use CreateEvaluation model
+        eval_data = CreateEvaluation(
+            name=name,
+            json_schema=json_schema,
+            project_id=project_id,
+            default_inference_settings=default_inference_settings,
+        )
+        return PreparedRequest(method="POST", url="/v1/evaluations", data=eval_data.model_dump(exclude_none=True, mode="json"))
+
+    def prepare_get(self, evaluation_id: str) -> PreparedRequest:
+        return PreparedRequest(method="GET", url=f"/v1/evaluations/{evaluation_id}")
+
+    def prepare_update(
+        self,
+        evaluation_id: str,
+        name: str = FieldUnset,
+        project_id: str = FieldUnset,
+        json_schema: dict[str, Any] = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> PreparedRequest:
+        """
+        Prepare a request to update an evaluation with partial updates.
+
+        Only the provided fields will be updated. Fields set to None will be excluded from the update.
+        """
+        # Build a dictionary with only the provided fields
+        data = PatchEvaluationRequest(
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            default_inference_settings=default_inference_settings,
+        ).model_dump(exclude_unset=True, mode="json")
+
+        return PreparedRequest(method="PATCH", url=f"/v1/evaluations/{evaluation_id}", data=data)
+
+    def prepare_list(self, project_id: str = FieldUnset) -> PreparedRequest:
+        """
+        Prepare a request to list evaluations.
+
+        Usage:
+        >>> client.evals.list(project_id="project_id")   # List all evaluations for a project
+        >>> client.evals.list()                          # List all evaluations (no project_id)
+
+        This does not work:
+        >>> client.evals.list(project_id=None)
+
+        Args:
+            project_id: The project ID to list evaluations for
+
+        Returns:
+            PreparedRequest: The prepared request
+        """
+        params = ListEvaluationParams(project_id=project_id).model_dump(exclude_unset=True, exclude_defaults=True, mode="json")
+        return PreparedRequest(method="GET", url="/v1/evaluations", params=params)
+
+    def prepare_delete(self, id: str) -> PreparedRequest:
+        return PreparedRequest(method="DELETE", url=f"/v1/evaluations/{id}")
+
+
+class Evaluations(SyncAPIResource, EvaluationsMixin):
+    """Evaluations API wrapper"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.documents = Documents(self._client)
+        self.iterations = Iterations(self._client)
+
+    def create(
+        self,
+        name: str,
+        json_schema: dict[str, Any],
+        project_id: str = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> Evaluation:
+        """
+        Create a new evaluation.
+
+        Args:
+            name: The name of the evaluation
+            json_schema: The JSON schema for the evaluation
+            project_id: The project ID to associate with the evaluation
+            documents: The documents to associate with the evaluation
+            default_inference_settings: The default inference settings to associate with the evaluation
+
+        Returns:
+            Evaluation: The created evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_create(name, json_schema, project_id, default_inference_settings=default_inference_settings)
+        response = self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    def get(self, evaluation_id: str) -> Evaluation:
+        """
+        Get an evaluation by ID.
+
+        Args:
+            evaluation_id: The ID of the evaluation to retrieve
+
+        Returns:
+            Evaluation: The evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_get(evaluation_id)
+        response = self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    def update(
+        self,
+        evaluation_id: str,
+        name: str = FieldUnset,
+        project_id: str = FieldUnset,
+        json_schema: dict[str, Any] = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> Evaluation:
+        """
+        Update an evaluation with partial updates.
+
+        Args:
+            evaluation_id: The ID of the evaluation to update
+            name: Optional new name for the evaluation
+            project_id: Optional new project ID
+            json_schema: Optional new JSON schema
+            documents: Optional list of documents to update
+            iterations: Optional list of iterations to update
+            default_inference_settings: Optional annotation properties
+
+        Returns:
+            Evaluation: The updated evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_update(
+            evaluation_id=evaluation_id,
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            default_inference_settings=default_inference_settings,
+        )
+        response = self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    def list(self, project_id: str = FieldUnset) -> List[Evaluation]:
+        """
+        List evaluations for a project.
+
+        Args:
+            project_id: The project ID to list evaluations for
+
+        Returns:
+            List[Evaluation]: List of evaluations
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_list(project_id)
+        response = self._client._prepared_request(request)
+        return [Evaluation(**item) for item in response.get("data", [])]
+
+    def delete(self, evaluation_id: str) -> DeleteResponse:
+        """
+        Delete an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation to delete
+
+        Returns:
+            DeleteResponse: The response containing success status and ID
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_delete(evaluation_id)
+        return self._client._prepared_request(request)
+
+
+class AsyncEvaluations(AsyncAPIResource, EvaluationsMixin):
+    """Async Evaluations API wrapper"""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.documents = AsyncDocuments(self._client)
+        self.iterations = AsyncIterations(self._client)
+
+    async def create(self, name: str, json_schema: Dict[str, Any], project_id: str = FieldUnset) -> Evaluation:
+        """
+        Create a new evaluation.
+
+        Args:
+            name: The name of the evaluation
+            json_schema: The JSON schema for the evaluation
+            project_id: The project ID to associate with the evaluation
+
+        Returns:
+            Evaluation: The created evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_create(name, json_schema, project_id=project_id)
+        response = await self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    async def get(self, evaluation_id: str) -> Evaluation:
+        """
+        Get an evaluation by ID.
+
+        Args:
+            evaluation_id: The ID of the evaluation to retrieve
+
+        Returns:
+            Evaluation: The evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_get(evaluation_id)
+        response = await self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    async def update(
+        self,
+        evaluation_id: str,
+        name: str = FieldUnset,
+        project_id: str = FieldUnset,
+        json_schema: dict[str, Any] = FieldUnset,
+        default_inference_settings: InferenceSettings = FieldUnset,
+    ) -> Evaluation:
+        """
+        Update an evaluation with partial updates.
+
+        Args:
+            id: The ID of the evaluation to update
+            name: Optional new name for the evaluation
+            project_id: Optional new project ID
+            json_schema: Optional new JSON schema
+            documents: Optional list of documents to update
+            iterations: Optional list of iterations to update
+            default_inference_settings: Optional annotation properties
+
+        Returns:
+            Evaluation: The updated evaluation
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_update(
+            evaluation_id=evaluation_id,
+            name=name,
+            project_id=project_id,
+            json_schema=json_schema,
+            default_inference_settings=default_inference_settings,
+        )
+        response = await self._client._prepared_request(request)
+        return Evaluation(**response)
+
+    async def list(self, project_id: str = FieldUnset) -> List[Evaluation]:
+        """
+        List evaluations for a project.
+
+        Args:
+            project_id: The project ID to list evaluations for
+
+        Returns:
+            List[Evaluation]: List of evaluations
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_list(project_id)
+        response = await self._client._prepared_request(request)
+        return [Evaluation(**item) for item in response.get("data", [])]
+
+    async def delete(self, evaluation_id: str) -> DeleteResponse:
+        """
+        Delete an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation to delete
+
+        Returns:
+            DeleteResponse: The response containing success status and ID
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_delete(evaluation_id)
+        return await self._client._prepared_request(request)

retab/resources/evaluations/documents.py

@@ -0,0 +1,233 @@
+from io import IOBase
+from pathlib import Path
+from typing import Any, Dict, List, Union
+
+import PIL.Image
+from pydantic import HttpUrl
+
+from ..._resource import AsyncAPIResource, SyncAPIResource
+from ..._utils.mime import prepare_mime_document
+from ...types.evaluations import DocumentItem, EvaluationDocument, PatchEvaluationDocumentRequest
+from ...types.mime import MIMEData
+from ...types.standards import PreparedRequest, DeleteResponse, FieldUnset
+from ...types.documents.extractions import UiParsedChatCompletion
+
+
+class DocumentsMixin:
+    def prepare_get(self, evaluation_id: str, document_id: str) -> PreparedRequest:
+        return PreparedRequest(method="GET", url=f"/v1/evaluations/{evaluation_id}/documents/{document_id}")
+
+    def prepare_create(self, evaluation_id: str, document: MIMEData, annotation: dict[str, Any]) -> PreparedRequest:
+        # Serialize the MIMEData
+        document_item = DocumentItem(mime_data=document, annotation=annotation, annotation_metadata=None)
+        return PreparedRequest(method="POST", url=f"/v1/evaluations/{evaluation_id}/documents", data=document_item.model_dump(mode="json"))
+
+    def prepare_list(self, evaluation_id: str) -> PreparedRequest:
+        return PreparedRequest(method="GET", url=f"/v1/evaluations/{evaluation_id}/documents")
+
+    def prepare_update(self, evaluation_id: str, document_id: str, annotation: dict[str, Any] = FieldUnset) -> PreparedRequest:
+        update_request = PatchEvaluationDocumentRequest(annotation=annotation)
+        return PreparedRequest(
+            method="PATCH", url=f"/v1/evaluations/{evaluation_id}/documents/{document_id}", data=update_request.model_dump(mode="json", exclude_unset=True, exclude_defaults=True)
+        )
+
+    def prepare_delete(self, evaluation_id: str, document_id: str) -> PreparedRequest:
+        return PreparedRequest(method="DELETE", url=f"/v1/evaluations/{evaluation_id}/documents/{document_id}")
+
+    def prepare_llm_annotate(self, evaluation_id: str, document_id: str) -> PreparedRequest:
+        return PreparedRequest(method="POST", url=f"/v1/evaluations/{evaluation_id}/documents/{document_id}/llm-annotate", data={"stream": False})
+
+
+class Documents(SyncAPIResource, DocumentsMixin):
+    """Documents API wrapper for evaluations"""
+
+    def create(self, evaluation_id: str, document: Union[Path, str, IOBase, MIMEData, PIL.Image.Image, HttpUrl], annotation: Dict[str, Any]) -> EvaluationDocument:
+        """
+        Create a document for an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+            document: The document to process. Can be:
+                - A file path (Path or str)
+                - A file-like object (IOBase)
+                - A MIMEData object
+                - A PIL Image object
+                - A URL (HttpUrl)
+            annotation: The ground truth for the document
+
+        Returns:
+            EvaluationDocument: The created document
+        Raises:
+            HTTPException if the request fails
+        """
+        # Convert document to MIME data format
+        mime_document: MIMEData = prepare_mime_document(document)
+
+        # Let prepare_create handle the serialization
+        request = self.prepare_create(evaluation_id, mime_document, annotation)
+        response = self._client._prepared_request(request)
+        return EvaluationDocument(**response)
+
+    def list(self, evaluation_id: str) -> List[EvaluationDocument]:
+        """
+        List documents for an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+
+        Returns:
+            List[EvaluationDocument]: List of documents
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_list(evaluation_id)
+        response = self._client._prepared_request(request)
+        return [EvaluationDocument(**item) for item in response.get("data", [])]
+
+    def get(self, evaluation_id: str, document_id: str) -> EvaluationDocument:
+        """
+        Get a document by ID.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+            document_id: The ID of the document
+
+        Returns:
+            EvaluationDocument: The document
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_get(evaluation_id, document_id)
+        response = self._client._prepared_request(request)
+        return EvaluationDocument(**response)
+
+    def update(self, evaluation_id: str, document_id: str, annotation: dict[str, Any] = FieldUnset) -> EvaluationDocument:
+        """
+        Update a document.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+            document_id: The ID of the document
+            annotation: The ground truth for the document
+        Returns:
+            EvaluationDocument: The updated document
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_update(evaluation_id, document_id, annotation=annotation)
+        response = self._client._prepared_request(request)
+        return EvaluationDocument(**response)
+
+    def delete(self, evaluation_id: str, document_id: str) -> DeleteResponse:
+        """
+        Delete a document.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+            document_id: The ID of the document
+
+        Returns:
+            DeleteResponse: The response containing success status and ID
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_delete(evaluation_id, document_id)
+        return self._client._prepared_request(request)
+
+    def llm_annotate(self, evaluation_id: str, document_id: str) -> UiParsedChatCompletion:
+        """
+        Annotate a document with an LLM. This method updates the document (within the evaluation) with the latest extraction.
+        """
+        request = self.prepare_llm_annotate(evaluation_id, document_id)
+        response = self._client._prepared_request(request)
+        return UiParsedChatCompletion(**response)
+
+
+class AsyncDocuments(AsyncAPIResource, DocumentsMixin):
+    """Async Documents API wrapper for evaluations"""
+
+    async def create(self, evaluation_id: str, document: Union[Path, str, IOBase, MIMEData, PIL.Image.Image, HttpUrl], annotation: Dict[str, Any]) -> EvaluationDocument:
+        """
+        Create a document for an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+            document: The document to process. Can be:
+                - A file path (Path or str)
+                - A file-like object (IOBase)
+                - A MIMEData object
+                - A PIL Image object
+                - A URL (HttpUrl)
+            annotation: The ground truth for the document
+
+        Returns:
+            EvaluationDocument: The created document
+        Raises:
+            HTTPException if the request fails
+        """
+        # Convert document to MIME data format
+        mime_document: MIMEData = prepare_mime_document(document)
+
+        # Let prepare_create handle the serialization
+        request = self.prepare_create(evaluation_id, mime_document, annotation)
+        response = await self._client._prepared_request(request)
+        return EvaluationDocument(**response)
+
+    async def list(self, evaluation_id: str) -> List[EvaluationDocument]:
+        """
+        List documents for an evaluation.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+
+        Returns:
+            List[EvaluationDocument]: List of documents
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_list(evaluation_id)
+        response = await self._client._prepared_request(request)
+        return [EvaluationDocument(**item) for item in response.get("data", [])]
+
+    async def update(self, evaluation_id: str, document_id: str, annotation: dict[str, Any] = FieldUnset) -> EvaluationDocument:
+        """
+        Update a document.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+            document_id: The ID of the document
+            annotation: The ground truth for the document
+
+        Returns:
+            EvaluationDocument: The updated document
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_update(evaluation_id, document_id, annotation)
+        response = await self._client._prepared_request(request)
+        return EvaluationDocument(**response)
+
+    async def delete(self, evaluation_id: str, document_id: str) -> DeleteResponse:
+        """
+        Delete a document.
+
+        Args:
+            evaluation_id: The ID of the evaluation
+            document_id: The ID of the document
+
+        Returns:
+            DeleteResponse: The response containing success status and ID
+        Raises:
+            HTTPException if the request fails
+        """
+        request = self.prepare_delete(evaluation_id, document_id)
+        return await self._client._prepared_request(request)
+
+    async def llm_annotate(self, evaluation_id: str, document_id: str) -> UiParsedChatCompletion:
+        """
+        Annotate a document with an LLM.
+        This method updates the document (within the evaluation) with the latest extraction.
+        """
+        request = self.prepare_llm_annotate(evaluation_id, document_id)
+        response = await self._client._prepared_request(request)
+        return UiParsedChatCompletion(**response)