latitude-sdk 2.1.2__py3-none-any.whl → 3.0.0__py3-none-any.whl

latitude_sdk/client/payloads.py

@@ -75,21 +75,17 @@ class EvaluationRequestParams(Model):
     conversation_uuid: str
 
 
-class TriggerEvaluationRequestParams(EvaluationRequestParams, Model):
-    pass
-
-
-class TriggerEvaluationRequestBody(Model):
-    evaluation_uuids: Optional[List[str]] = Field(default=None, alias=str("evaluationUuids"))
+class AnnotateEvaluationRequestParams(EvaluationRequestParams, Model):
+    evaluation_uuid: str
 
 
-class CreateEvaluationResultRequestParams(EvaluationRequestParams, Model):
-    evaluation_uuid: str
+class AnnotateEvaluationRequestBody(Model):
+    score: int
 
+    class Metadata(Model):
+        reason: str
 
-class CreateEvaluationResultRequestBody(Model):
-    result: Union[str, bool, int]
-    reason: str
+    metadata: Optional[Metadata] = None
 
 
 RequestParams = Union[
@@ -99,8 +95,7 @@ RequestParams = Union[
     RunPromptRequestParams,
     ChatPromptRequestParams,
     CreateLogRequestParams,
-    TriggerEvaluationRequestParams,
-    CreateEvaluationResultRequestParams,
+    AnnotateEvaluationRequestParams,
 ]
 
 
@@ -109,8 +104,7 @@ RequestBody = Union[
     RunPromptRequestBody,
     ChatPromptRequestBody,
     CreateLogRequestBody,
-    TriggerEvaluationRequestBody,
-    CreateEvaluationResultRequestBody,
+    AnnotateEvaluationRequestBody,
 ]
 
 
@@ -121,5 +115,4 @@ class RequestHandler(StrEnum):
     RunPrompt = "RUN_PROMPT"
     ChatPrompt = "CHAT_PROMPT"
     CreateLog = "CREATE_LOG"
-    TriggerEvaluation = "TRIGGER_EVALUATION"
-    CreateEvaluationResult = "CREATE_EVALUATION_RESULT"
+    AnnotateEvaluation = "ANNOTATE_EVALUATION"

latitude_sdk/client/router.py

@@ -1,8 +1,8 @@
 from typing import Callable, Optional, Tuple
 
 from latitude_sdk.client.payloads import (
+    AnnotateEvaluationRequestParams,
     ChatPromptRequestParams,
-    CreateEvaluationResultRequestParams,
     CreateLogRequestParams,
     GetAllPromptRequestParams,
     GetOrCreatePromptRequestParams,
@@ -10,7 +10,6 @@ from latitude_sdk.client.payloads import (
     RequestHandler,
     RequestParams,
     RunPromptRequestParams,
-    TriggerEvaluationRequestParams,
 )
 from latitude_sdk.sdk.types import GatewayOptions
 from latitude_sdk.util import Model
@@ -40,26 +39,35 @@ class Router:
         if handler == RequestHandler.GetAllPrompts:
             assert isinstance(params, GetAllPromptRequestParams)
 
-            return "GET", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).all_prompts
+            return (
+                "GET",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).all_prompts,
+            )
 
         elif handler == RequestHandler.GetOrCreatePrompt:
             assert isinstance(params, GetOrCreatePromptRequestParams)
 
-            return "POST", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).get_or_create
+            return (
+                "POST",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).get_or_create,
+            )
 
         elif handler == RequestHandler.RunPrompt:
             assert isinstance(params, RunPromptRequestParams)
 
-            return "POST", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).run
+            return (
+                "POST",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).run,
+            )
 
         elif handler == RequestHandler.ChatPrompt:
             assert isinstance(params, ChatPromptRequestParams)
@@ -69,36 +77,31 @@ class Router:
         elif handler == RequestHandler.CreateLog:
             assert isinstance(params, CreateLogRequestParams)
 
-            return "POST", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).logs
-
-        elif handler == RequestHandler.TriggerEvaluation:
-            assert isinstance(params, TriggerEvaluationRequestParams)
-
-            return "POST", self.conversations().evaluate(params.conversation_uuid)
+            return (
+                "POST",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).logs,
+            )
 
-        elif handler == RequestHandler.CreateEvaluationResult:
-            assert isinstance(params, CreateEvaluationResultRequestParams)
+        elif handler == RequestHandler.AnnotateEvaluation:
+            assert isinstance(params, AnnotateEvaluationRequestParams)
 
-            return "POST", self.conversations().evaluation_result(params.conversation_uuid, params.evaluation_uuid)
+            return "POST", self.conversations().annotate(params.conversation_uuid, params.evaluation_uuid)
 
         raise TypeError(f"Unknown handler: {handler}")
 
     class Conversations(Model):
         chat: Callable[[str], str]
-        evaluate: Callable[[str], str]
-        evaluation_result: Callable[[str, str], str]
+        annotate: Callable[[str, str], str]
 
     def conversations(self) -> Conversations:
         base_url = f"{self.options.gateway.base_url}/conversations"
 
         return self.Conversations(
             chat=lambda uuid: f"{base_url}/{uuid}/chat",
-            evaluate=lambda uuid: f"{base_url}/{uuid}/evaluate",
-            evaluation_result=lambda conversation_uuid,
-            evaluation_uuid: f"{base_url}/{conversation_uuid}/evaluations/{evaluation_uuid}/evaluation-results",
+            annotate=lambda uuid, evaluation_uuid: f"{base_url}/{uuid}/evaluations/{evaluation_uuid}/annotate",
         )
 
     class Prompts(Model):

latitude_sdk/sdk/evaluations.py

@@ -1,32 +1,30 @@
-from typing import List, Optional, Union
+from datetime import datetime
+from typing import Any, Optional, Union
 
 from latitude_sdk.client import (
+    AnnotateEvaluationRequestBody,
+    AnnotateEvaluationRequestParams,
     Client,
-    CreateEvaluationResultRequestBody,
-    CreateEvaluationResultRequestParams,
     RequestHandler,
-    TriggerEvaluationRequestBody,
-    TriggerEvaluationRequestParams,
 )
-from latitude_sdk.sdk.types import EvaluationResult, SdkOptions
-from latitude_sdk.util import Model
+from latitude_sdk.sdk.types import SdkOptions
+from latitude_sdk.util import Field, Model
 
 
-class TriggerEvaluationOptions(Model):
-    evaluation_uuids: Optional[List[str]] = None
-
-
-class TriggerEvaluationResult(Model):
-    evaluations: List[str]
-
-
-class CreateEvaluationResultOptions(Model):
-    result: Union[str, bool, int]
+class AnnotateEvaluationOptions(Model):
     reason: str
 
 
-class CreateEvaluationResultResult(EvaluationResult, Model):
-    pass
+class AnnotateEvaluationResult(Model):
+    uuid: str
+    score: int
+    normalized_score: int = Field(alias=str("normalizedScore"))
+    metadata: dict[str, Any]
+    has_passed: bool = Field(alias=str("hasPassed"))
+    created_at: datetime = Field(alias=str("createdAt"))
+    updated_at: datetime = Field(alias=str("updatedAt"))
+    version_uuid: str = Field(alias=str("versionUuid"))
+    error: Optional[Union[str, None]] = None
 
 
 class Evaluations:
@@ -37,34 +35,24 @@ class Evaluations:
         self._options = options
         self._client = client
 
-    async def trigger(self, uuid: str, options: Optional[TriggerEvaluationOptions] = None) -> TriggerEvaluationResult:
-        options = TriggerEvaluationOptions(**{**dict(self._options), **dict(options or {})})
-
-        async with self._client.request(
-            handler=RequestHandler.TriggerEvaluation,
-            params=TriggerEvaluationRequestParams(
-                conversation_uuid=uuid,
-            ),
-            body=TriggerEvaluationRequestBody(
-                evaluation_uuids=options.evaluation_uuids,
-            ),
-        ) as response:
-            return TriggerEvaluationResult.model_validate_json(response.content)
-
-    async def create_result(
-        self, uuid: str, evaluation_uuid: str, options: CreateEvaluationResultOptions
-    ) -> CreateEvaluationResultResult:
-        options = CreateEvaluationResultOptions(**{**dict(self._options), **dict(options)})
+    async def annotate(
+        self,
+        uuid: str,
+        evaluation_uuid: str,
+        score: int,
+        options: Optional[AnnotateEvaluationOptions] = None,
+    ) -> AnnotateEvaluationResult:
+        options = AnnotateEvaluationOptions(**{**dict(self._options), **dict(options or {})})
 
         async with self._client.request(
-            handler=RequestHandler.CreateEvaluationResult,
-            params=CreateEvaluationResultRequestParams(
+            handler=RequestHandler.AnnotateEvaluation,
+            params=AnnotateEvaluationRequestParams(
                 conversation_uuid=uuid,
                 evaluation_uuid=evaluation_uuid,
             ),
-            body=CreateEvaluationResultRequestBody(
-                result=options.result,
-                reason=options.reason,
+            body=AnnotateEvaluationRequestBody(
+                score=score,
+                metadata=(AnnotateEvaluationRequestBody.Metadata(reason=options.reason) if options.reason else None),
             ),
         ) as response:
-            return CreateEvaluationResultResult.model_validate_json(response.content)
+            return AnnotateEvaluationResult.model_validate_json(response.content)

latitude_sdk/sdk/types.py

@@ -1,5 +1,15 @@
 from datetime import datetime
-from typing import Any, Callable, List, Literal, Optional, Protocol, Sequence, Union, runtime_checkable
+from typing import (
+    Any,
+    Callable,
+    List,
+    Literal,
+    Optional,
+    Protocol,
+    Sequence,
+    Union,
+    runtime_checkable,
+)
 
 from promptl_ai import Message, MessageLike
 
@@ -230,28 +240,6 @@ class Log(Model):
     updated_at: datetime = Field(alias=str("updatedAt"))
 
 
-class EvaluationResultType(StrEnum):
-    Boolean = "evaluation_resultable_booleans"
-    Text = "evaluation_resultable_texts"
-    Number = "evaluation_resultable_numbers"
-
-
-class EvaluationResult(Model):
-    id: int
-    uuid: str
-    evaluation_id: int = Field(alias=str("evaluationId"))
-    document_log_id: int = Field(alias=str("documentLogId"))
-    evaluated_provider_log_id: Optional[int] = Field(default=None, alias=str("evaluatedProviderLogId"))
-    evaluation_provider_log_id: Optional[int] = Field(default=None, alias=str("evaluationProviderLogId"))
-    resultable_type: Optional[EvaluationResultType] = Field(default=None, alias=str("resultableType"))
-    resultable_id: Optional[int] = Field(default=None, alias=str("resultableId"))
-    result: Optional[Union[str, bool, int]] = None
-    source: Optional[LogSources] = None
-    reason: Optional[str] = None
-    created_at: datetime = Field(alias=str("createdAt"))
-    updated_at: datetime = Field(alias=str("updatedAt"))
-
-
 class StreamCallbacks(Model):
     @runtime_checkable
     class OnEvent(Protocol):

{latitude_sdk-2.1.2.dist-info → latitude_sdk-3.0.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: latitude-sdk
-Version: 2.1.2
+Version: 3.0.0
 Summary: Latitude SDK for Python
 Project-URL: repository, https://github.com/latitude-dev/latitude-llm/tree/main/packages/sdks/python
 Project-URL: homepage, https://github.com/latitude-dev/latitude-llm/tree/main/packages/sdks/python#readme

{latitude_sdk-2.1.2.dist-info → latitude_sdk-3.0.0.dist-info}/RECORD

@@ -2,19 +2,19 @@ latitude_sdk/__init__.py,sha256=-AbNXLmzDZeGbRdDIOpNjdCbacOvLBflSJwQtLlZfgk,19
 latitude_sdk/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 latitude_sdk/client/__init__.py,sha256=d8CnNB8UoGwcftiIeeC0twdg01qNvfpj-v7O40I7IiE,68
 latitude_sdk/client/client.py,sha256=Tlg0IU7_64O9yo0BfQEdVwVsjp-5_DUwJZqlX3uEb0w,4399
-latitude_sdk/client/payloads.py,sha256=GCMX9my5lj9bzTQNbgs3jcr9M0b-axt5g9xqey2bnSs,2860
-latitude_sdk/client/router.py,sha256=CiJnhXDJaqMh7H_bG4bOX-dlrELcpHVH_8l0ZRz7UZo,4414
+latitude_sdk/client/payloads.py,sha256=XiGadgaZhCMKa2auBPLln6RTEt8RwEqx_H8FoSv5h2c,2554
+latitude_sdk/client/router.py,sha256=sdNvUNXqoM5TDmvN7toyJWuI8Lt4Z6MhKmWWKitEFes,4218
 latitude_sdk/env/__init__.py,sha256=66of5veJ-u1aNI025L65Rrj321AjrYevMqomTMYIrPQ,19
 latitude_sdk/env/env.py,sha256=MnXexPOHE6aXcAszrDCbW7hzACUv4YtU1bfxpYwvHNw,455
 latitude_sdk/sdk/__init__.py,sha256=C9LlIjfnrS7KOK3-ruXKmbT77nSQMm23nZ6-t8sO8ME,137
 latitude_sdk/sdk/errors.py,sha256=9GlGdDE8LGy3dE2Ry_BipBg-tDbQx7LWXJfSnTJSSBE,1747
-latitude_sdk/sdk/evaluations.py,sha256=fDGtAWjdPG9OuKLit6u-jufVleC1EnshRplK6RN8iyg,2277
+latitude_sdk/sdk/evaluations.py,sha256=yZPZRLXfaqnNop6axhMPB-Izj4AW98jvBhXBZfH-2qM,1845
 latitude_sdk/sdk/latitude.py,sha256=lUlGOiZXSFt0zm3sTHfBgjKzbcVJueMk6MXTGn9WCn8,2570
 latitude_sdk/sdk/logs.py,sha256=CyHkRJvPl_p7wTSvR9bgxEI5akS0Tjc9FeQRb2C2vMg,1997
 latitude_sdk/sdk/prompts.py,sha256=WztCpDSt0mxng0N2hyIrrzEw1TanqGRIl7Cpc_5ei4M,17369
-latitude_sdk/sdk/types.py,sha256=wGxwFEzHWShy6a2eGUt4NwRkKP_snvSrlH0rYwNYdgY,8568
+latitude_sdk/sdk/types.py,sha256=e_AQ6YGRNgYvPJOxD5rndH5zPtslKMBhBtLocWlIHQE,7626
 latitude_sdk/util/__init__.py,sha256=alIDGBnxWH4JvP-UW-7N99seBBi0r1GV1h8f1ERFBec,21
 latitude_sdk/util/utils.py,sha256=hMOmF-u1QaDgOwXN6ME6n4TaQ70yZKLvijDUqNCMwXI,2844
-latitude_sdk-2.1.2.dist-info/METADATA,sha256=Af-YXAT2lZOY_cNR2ZhL1KpgkTdtRts8g2BEsimo4e4,2327
-latitude_sdk-2.1.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-latitude_sdk-2.1.2.dist-info/RECORD,,
+latitude_sdk-3.0.0.dist-info/METADATA,sha256=ibP2KsIYVUvIMBkyLgFxJ8kDqiP4QA-z9YAy-Olj5fE,2327
+latitude_sdk-3.0.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+latitude_sdk-3.0.0.dist-info/RECORD,,