latitude-sdk 2.1.2__tar.gz → 3.0.0__tar.gz

{latitude_sdk-2.1.2 → latitude_sdk-3.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: latitude-sdk
-Version: 2.1.2
+Version: 3.0.0
 Summary: Latitude SDK for Python
 Project-URL: repository, https://github.com/latitude-dev/latitude-llm/tree/main/packages/sdks/python
 Project-URL: homepage, https://github.com/latitude-dev/latitude-llm/tree/main/packages/sdks/python#readme

{latitude_sdk-2.1.2 → latitude_sdk-3.0.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "latitude-sdk"
-version = "2.1.2"
+version = "3.0.0"
 description = "Latitude SDK for Python"
 authors = [{ name = "Latitude Data SL", email = "hello@latitude.so" }]
 maintainers = [{ name = "Latitude Data SL", email = "hello@latitude.so" }]

{latitude_sdk-2.1.2 → latitude_sdk-3.0.0}/src/latitude_sdk/client/payloads.py

@@ -75,21 +75,17 @@ class EvaluationRequestParams(Model):
     conversation_uuid: str
 
 
-class TriggerEvaluationRequestParams(EvaluationRequestParams, Model):
-    pass
-
-
-class TriggerEvaluationRequestBody(Model):
-    evaluation_uuids: Optional[List[str]] = Field(default=None, alias=str("evaluationUuids"))
+class AnnotateEvaluationRequestParams(EvaluationRequestParams, Model):
+    evaluation_uuid: str
 
 
-class CreateEvaluationResultRequestParams(EvaluationRequestParams, Model):
-    evaluation_uuid: str
+class AnnotateEvaluationRequestBody(Model):
+    score: int
 
+    class Metadata(Model):
+        reason: str
 
-class CreateEvaluationResultRequestBody(Model):
-    result: Union[str, bool, int]
-    reason: str
+    metadata: Optional[Metadata] = None
 
 
 RequestParams = Union[
@@ -99,8 +95,7 @@ RequestParams = Union[
     RunPromptRequestParams,
     ChatPromptRequestParams,
     CreateLogRequestParams,
-    TriggerEvaluationRequestParams,
-    CreateEvaluationResultRequestParams,
+    AnnotateEvaluationRequestParams,
 ]
 
 
@@ -109,8 +104,7 @@ RequestBody = Union[
     RunPromptRequestBody,
     ChatPromptRequestBody,
     CreateLogRequestBody,
-    TriggerEvaluationRequestBody,
-    CreateEvaluationResultRequestBody,
+    AnnotateEvaluationRequestBody,
 ]
 
 
@@ -121,5 +115,4 @@ class RequestHandler(StrEnum):
     RunPrompt = "RUN_PROMPT"
     ChatPrompt = "CHAT_PROMPT"
     CreateLog = "CREATE_LOG"
-    TriggerEvaluation = "TRIGGER_EVALUATION"
-    CreateEvaluationResult = "CREATE_EVALUATION_RESULT"
+    AnnotateEvaluation = "ANNOTATE_EVALUATION"

{latitude_sdk-2.1.2 → latitude_sdk-3.0.0}/src/latitude_sdk/client/router.py

@@ -1,8 +1,8 @@
 from typing import Callable, Optional, Tuple
 
 from latitude_sdk.client.payloads import (
+    AnnotateEvaluationRequestParams,
     ChatPromptRequestParams,
-    CreateEvaluationResultRequestParams,
     CreateLogRequestParams,
     GetAllPromptRequestParams,
     GetOrCreatePromptRequestParams,
@@ -10,7 +10,6 @@ from latitude_sdk.client.payloads import (
     RequestHandler,
     RequestParams,
     RunPromptRequestParams,
-    TriggerEvaluationRequestParams,
 )
 from latitude_sdk.sdk.types import GatewayOptions
 from latitude_sdk.util import Model
@@ -40,26 +39,35 @@ class Router:
         if handler == RequestHandler.GetAllPrompts:
             assert isinstance(params, GetAllPromptRequestParams)
 
-            return "GET", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).all_prompts
+            return (
+                "GET",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).all_prompts,
+            )
 
         elif handler == RequestHandler.GetOrCreatePrompt:
             assert isinstance(params, GetOrCreatePromptRequestParams)
 
-            return "POST", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).get_or_create
+            return (
+                "POST",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).get_or_create,
+            )
 
         elif handler == RequestHandler.RunPrompt:
             assert isinstance(params, RunPromptRequestParams)
 
-            return "POST", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).run
+            return (
+                "POST",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).run,
+            )
 
         elif handler == RequestHandler.ChatPrompt:
             assert isinstance(params, ChatPromptRequestParams)
@@ -69,36 +77,31 @@ class Router:
         elif handler == RequestHandler.CreateLog:
             assert isinstance(params, CreateLogRequestParams)
 
-            return "POST", self.prompts(
-                project_id=params.project_id,
-                version_uuid=params.version_uuid,
-            ).logs
-
-        elif handler == RequestHandler.TriggerEvaluation:
-            assert isinstance(params, TriggerEvaluationRequestParams)
-
-            return "POST", self.conversations().evaluate(params.conversation_uuid)
+            return (
+                "POST",
+                self.prompts(
+                    project_id=params.project_id,
+                    version_uuid=params.version_uuid,
+                ).logs,
+            )
 
-        elif handler == RequestHandler.CreateEvaluationResult:
-            assert isinstance(params, CreateEvaluationResultRequestParams)
+        elif handler == RequestHandler.AnnotateEvaluation:
+            assert isinstance(params, AnnotateEvaluationRequestParams)
 
-            return "POST", self.conversations().evaluation_result(params.conversation_uuid, params.evaluation_uuid)
+            return "POST", self.conversations().annotate(params.conversation_uuid, params.evaluation_uuid)
 
         raise TypeError(f"Unknown handler: {handler}")
 
     class Conversations(Model):
         chat: Callable[[str], str]
-        evaluate: Callable[[str], str]
-        evaluation_result: Callable[[str, str], str]
+        annotate: Callable[[str, str], str]
 
     def conversations(self) -> Conversations:
         base_url = f"{self.options.gateway.base_url}/conversations"
 
         return self.Conversations(
             chat=lambda uuid: f"{base_url}/{uuid}/chat",
-            evaluate=lambda uuid: f"{base_url}/{uuid}/evaluate",
-            evaluation_result=lambda conversation_uuid,
-            evaluation_uuid: f"{base_url}/{conversation_uuid}/evaluations/{evaluation_uuid}/evaluation-results",
+            annotate=lambda uuid, evaluation_uuid: f"{base_url}/{uuid}/evaluations/{evaluation_uuid}/annotate",
         )
 
     class Prompts(Model):

latitude_sdk-3.0.0/src/latitude_sdk/sdk/evaluations.py

@@ -0,0 +1,58 @@
+from datetime import datetime
+from typing import Any, Optional, Union
+
+from latitude_sdk.client import (
+    AnnotateEvaluationRequestBody,
+    AnnotateEvaluationRequestParams,
+    Client,
+    RequestHandler,
+)
+from latitude_sdk.sdk.types import SdkOptions
+from latitude_sdk.util import Field, Model
+
+
+class AnnotateEvaluationOptions(Model):
+    reason: str
+
+
+class AnnotateEvaluationResult(Model):
+    uuid: str
+    score: int
+    normalized_score: int = Field(alias=str("normalizedScore"))
+    metadata: dict[str, Any]
+    has_passed: bool = Field(alias=str("hasPassed"))
+    created_at: datetime = Field(alias=str("createdAt"))
+    updated_at: datetime = Field(alias=str("updatedAt"))
+    version_uuid: str = Field(alias=str("versionUuid"))
+    error: Optional[Union[str, None]] = None
+
+
+class Evaluations:
+    _options: SdkOptions
+    _client: Client
+
+    def __init__(self, client: Client, options: SdkOptions):
+        self._options = options
+        self._client = client
+
+    async def annotate(
+        self,
+        uuid: str,
+        evaluation_uuid: str,
+        score: int,
+        options: Optional[AnnotateEvaluationOptions] = None,
+    ) -> AnnotateEvaluationResult:
+        options = AnnotateEvaluationOptions(**{**dict(self._options), **dict(options or {})})
+
+        async with self._client.request(
+            handler=RequestHandler.AnnotateEvaluation,
+            params=AnnotateEvaluationRequestParams(
+                conversation_uuid=uuid,
+                evaluation_uuid=evaluation_uuid,
+            ),
+            body=AnnotateEvaluationRequestBody(
+                score=score,
+                metadata=(AnnotateEvaluationRequestBody.Metadata(reason=options.reason) if options.reason else None),
+            ),
+        ) as response:
+            return AnnotateEvaluationResult.model_validate_json(response.content)

{latitude_sdk-2.1.2 → latitude_sdk-3.0.0}/src/latitude_sdk/sdk/types.py

@@ -1,5 +1,15 @@
 from datetime import datetime
-from typing import Any, Callable, List, Literal, Optional, Protocol, Sequence, Union, runtime_checkable
+from typing import (
+    Any,
+    Callable,
+    List,
+    Literal,
+    Optional,
+    Protocol,
+    Sequence,
+    Union,
+    runtime_checkable,
+)
 
 from promptl_ai import Message, MessageLike
 
@@ -230,28 +240,6 @@ class Log(Model):
     updated_at: datetime = Field(alias=str("updatedAt"))
 
 
-class EvaluationResultType(StrEnum):
-    Boolean = "evaluation_resultable_booleans"
-    Text = "evaluation_resultable_texts"
-    Number = "evaluation_resultable_numbers"
-
-
-class EvaluationResult(Model):
-    id: int
-    uuid: str
-    evaluation_id: int = Field(alias=str("evaluationId"))
-    document_log_id: int = Field(alias=str("documentLogId"))
-    evaluated_provider_log_id: Optional[int] = Field(default=None, alias=str("evaluatedProviderLogId"))
-    evaluation_provider_log_id: Optional[int] = Field(default=None, alias=str("evaluationProviderLogId"))
-    resultable_type: Optional[EvaluationResultType] = Field(default=None, alias=str("resultableType"))
-    resultable_id: Optional[int] = Field(default=None, alias=str("resultableId"))
-    result: Optional[Union[str, bool, int]] = None
-    source: Optional[LogSources] = None
-    reason: Optional[str] = None
-    created_at: datetime = Field(alias=str("createdAt"))
-    updated_at: datetime = Field(alias=str("updatedAt"))
-
-
 class StreamCallbacks(Model):
     @runtime_checkable
     class OnEvent(Protocol):

latitude_sdk-2.1.2/tests/evaluations/create_result_test.py → latitude_sdk-3.0.0/tests/evaluations/annotate_test.py

@@ -2,7 +2,7 @@ from typing import List, cast
 
 import httpx
 
-from latitude_sdk import CreateEvaluationResultOptions, CreateEvaluationResultResult
+from latitude_sdk import AnnotateEvaluationOptions, AnnotateEvaluationResult
 from tests.utils import TestCase, fixtures
 
 
@@ -10,13 +10,13 @@ class TestCreateEvaluationResult(TestCase):
     async def test_success(self):
         conversation_uuid = "conversation-uuid"
         evaluation_uuid = "evaluation-uuid"
-        options = CreateEvaluationResultOptions(result=True, reason="Because Yes")
-        endpoint = f"/conversations/{conversation_uuid}/evaluations/{evaluation_uuid}/evaluation-results"
+        options = AnnotateEvaluationOptions(reason="Because Yes")
+        endpoint = f"/conversations/{conversation_uuid}/evaluations/{evaluation_uuid}/annotate"
         endpoint_mock = self.gateway_mock.post(endpoint).mock(
             return_value=httpx.Response(200, json=fixtures.EVALUATION_RESULT_RESPONSE)
         )
 
-        result = await self.sdk.evaluations.create_result(conversation_uuid, evaluation_uuid, options)
+        result = await self.sdk.evaluations.annotate(conversation_uuid, evaluation_uuid, 1, options)
         request, _ = endpoint_mock.calls.last
 
         self.assert_requested(
@@ -24,24 +24,27 @@ class TestCreateEvaluationResult(TestCase):
             method="POST",
             endpoint=endpoint,
             body={
-                "result": options.result,
-                "reason": options.reason,
+                "score": 1,
+                "metadata": {"reason": options.reason},
             },
         )
         self.assertEqual(endpoint_mock.call_count, 1)
-        self.assertEqual(result, CreateEvaluationResultResult(**dict(fixtures.EVALUATION_RESULT)))
+        self.assertEqual(
+            result,
+            AnnotateEvaluationResult(**dict(fixtures.EVALUATION_RESULT)),
+        )
 
     async def test_fails(self):
         conversation_uuid = "conversation-uuid"
         evaluation_uuid = "evaluation-uuid"
-        options = CreateEvaluationResultOptions(result=True, reason="Because Yes")
-        endpoint = f"/conversations/{conversation_uuid}/evaluations/{evaluation_uuid}/evaluation-results"
+        options = AnnotateEvaluationOptions(reason="Because Yes")
+        endpoint = f"/conversations/{conversation_uuid}/evaluations/{evaluation_uuid}/annotate"
         endpoint_mock = self.gateway_mock.post(endpoint).mock(
             return_value=httpx.Response(500, json=fixtures.ERROR_RESPONSE)
         )
 
         with self.assertRaisesRegex(type(fixtures.ERROR), fixtures.ERROR.message):
-            await self.sdk.evaluations.create_result(conversation_uuid, evaluation_uuid, options)
+            await self.sdk.evaluations.annotate(conversation_uuid, evaluation_uuid, 1, options)
         requests = cast(List[httpx.Request], [request for request, _ in endpoint_mock.calls])  # type: ignore
 
         [
@@ -50,8 +53,8 @@ class TestCreateEvaluationResult(TestCase):
                 method="POST",
                 endpoint=endpoint,
                 body={
-                    "result": options.result,
-                    "reason": options.reason,
+                    "score": 1,
+                    "metadata": {"reason": options.reason},
                 },
             )
             for request in requests

{latitude_sdk-2.1.2 → latitude_sdk-3.0.0}/tests/utils/fixtures.py

@@ -16,6 +16,7 @@ from promptl_ai import (
 
 from latitude_sdk import (
     AGENT_END_TOOL_NAME,
+    AnnotateEvaluationResult,
     ApiError,
     ApiErrorCodes,
     ChainError,
@@ -29,8 +30,6 @@ from latitude_sdk import (
     ChainEventStepStarted,
     ChainEventToolsRequested,
     ChainTextResponse,
-    EvaluationResult,
-    EvaluationResultType,
     FinishedResult,
     FinishReason,
     Log,
@@ -173,36 +172,28 @@ EVALUATIONS = [
 ]
 
 EVALUATION_RESULT_RESPONSE: dict[str, Any] = {
-    "id": 31,
     "uuid": "e25a317b-c682-4c25-a704-a87ac79507c4",
-    "evaluationId": 31,
-    "documentLogId": 31,
-    "evaluatedProviderLogId": 31,
-    "evaluationProviderLogId": None,
-    "resultableType": "evaluation_resultable_booleans",
-    "resultableId": 31,
-    "result": True,
-    "source": "api",
-    "reason": "Because Yes",
+    "score": 1,
+    "normalizedScore": 1,
+    "metadata": {"reason": "Because Yes"},
+    "hasPassed": True,
     "createdAt": "2025-01-01 00:00:00.000",
     "updatedAt": "2025-01-01 00:00:00.000",
+    "versionUuid": "e25a317b-c682-4c25-a704-a87ac79507c4",
+    "error": None,
 }
 
 
-EVALUATION_RESULT = EvaluationResult(
-    id=31,
+EVALUATION_RESULT = AnnotateEvaluationResult(
     uuid="e25a317b-c682-4c25-a704-a87ac79507c4",
-    evaluation_id=31,
-    document_log_id=31,
-    evaluated_provider_log_id=31,
-    evaluation_provider_log_id=None,
-    resultable_type=EvaluationResultType.Boolean,
-    resultable_id=31,
-    result=True,
-    source=LogSources.Api,
-    reason="Because Yes",
+    score=1,
+    normalized_score=1,
+    metadata={"reason": "Because Yes"},
+    has_passed=True,
     created_at=datetime(2025, 1, 1, 0, 0, 0, 0),
     updated_at=datetime(2025, 1, 1, 0, 0, 0, 0),
+    version_uuid="e25a317b-c682-4c25-a704-a87ac79507c4",
+    error=None,
 )
 
 CONVERSATION_EVENTS_STREAM: list[str] = [
@@ -843,7 +834,10 @@ CONVERSATION_EVENTS: list[StreamEvent] = [
         "finishReason": "stop",
         "isContinued": False,
         "experimental_providerMetadata": {"openai": {"reasoningTokens": 0, "cachedPromptTokens": 0}},
-        "response": {"timestamp": "2025-01-02T12:29:13.000Z", "modelId": "gpt-4o-mini-latest"},
+        "response": {
+            "timestamp": "2025-01-02T12:29:13.000Z",
+            "modelId": "gpt-4o-mini-latest",
+        },
         "usage": {"promptTokens": 31, "completionTokens": 9, "totalTokens": 40},
     },
     {
@@ -851,7 +845,10 @@ CONVERSATION_EVENTS: list[StreamEvent] = [
         "type": "finish",
         "finishReason": "stop",
         "experimental_providerMetadata": {"openai": {"reasoningTokens": 0, "cachedPromptTokens": 0}},
-        "response": {"timestamp": "2025-01-02T12:29:13.000Z", "modelId": "gpt-4o-mini-latest"},
+        "response": {
+            "timestamp": "2025-01-02T12:29:13.000Z",
+            "modelId": "gpt-4o-mini-latest",
+        },
         "usage": {"promptTokens": 31, "completionTokens": 9, "totalTokens": 40},
     },
     ChainEventProviderCompleted(
@@ -956,7 +953,10 @@ CONVERSATION_EVENTS: list[StreamEvent] = [
         "finishReason": "tool-calls",
         "isContinued": False,
         "experimental_providerMetadata": {"openai": {"reasoningTokens": 0, "cachedPromptTokens": 0}},
-        "response": {"timestamp": "2025-01-02T12:29:16.000Z", "modelId": "gpt-4o-mini-latest"},
+        "response": {
+            "timestamp": "2025-01-02T12:29:16.000Z",
+            "modelId": "gpt-4o-mini-latest",
+        },
         "usage": {"promptTokens": 61, "completionTokens": 9, "totalTokens": 70},
     },
     {
@@ -964,7 +964,10 @@ CONVERSATION_EVENTS: list[StreamEvent] = [
         "type": "finish",
         "finishReason": "tool-calls",
         "experimental_providerMetadata": {"openai": {"reasoningTokens": 0, "cachedPromptTokens": 0}},
-        "response": {"timestamp": "2025-01-02T12:29:16.000Z", "modelId": "gpt-4o-mini-latest"},
+        "response": {
+            "timestamp": "2025-01-02T12:29:16.000Z",
+            "modelId": "gpt-4o-mini-latest",
+        },
         "usage": {"promptTokens": 61, "completionTokens": 9, "totalTokens": 70},
     },
     ChainEventProviderCompleted(
@@ -1033,9 +1036,15 @@ CONVERSATION_EVENTS: list[StreamEvent] = [
             AssistantMessage(content=[TextContent(text="I should look at their decimals.")]),
         ],
         tools=[
-            ToolCall(id="toolu_01ARatRfRidTDshkg1UuQhW2", name="calculator", arguments={"expression": "9.9 > 9.11"}),
             ToolCall(
-                id="toolu_B0398l23AOdTDshkg1UuQhZ3", name="calculator", arguments={"expression": "9.9 less than 9.11"}
+                id="toolu_01ARatRfRidTDshkg1UuQhW2",
+                name="calculator",
+                arguments={"expression": "9.9 > 9.11"},
+            ),
+            ToolCall(
+                id="toolu_B0398l23AOdTDshkg1UuQhZ3",
+                name="calculator",
+                arguments={"expression": "9.9 less than 9.11"},
             ),
             ToolCall(
                 id="toolu_K12398312kjadbsadZ77JAS4",
@@ -1173,11 +1182,26 @@ CONVERSATION_FINISHED_RESULT_RESPONSE: dict[str, Any] = {
         },
     ],
     "conversation": [
-        {"role": "system", "content": [{"type": "text", "text": "Reason before answering."}]},
-        {"role": "user", "content": [{"type": "text", "text": "My question is: Is 9.9 greater than 9.11?"}]},
-        {"role": "assistant", "content": [{"type": "text", "text": "I should look at their decimals."}]},
-        {"role": "system", "content": [{"type": "text", "text": "Now answer succinctly."}]},
-        {"role": "user", "content": [{"type": "text", "text": "My question was: Is 9.9 greater than 9.11?"}]},
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": "Reason before answering."}],
+        },
+        {
+            "role": "user",
+            "content": [{"type": "text", "text": "My question is: Is 9.9 greater than 9.11?"}],
+        },
+        {
+            "role": "assistant",
+            "content": [{"type": "text", "text": "I should look at their decimals."}],
+        },
+        {
+            "role": "system",
+            "content": [{"type": "text", "text": "Now answer succinctly."}],
+        },
+        {
+            "role": "user",
+            "content": [{"type": "text", "text": "My question was: Is 9.9 greater than 9.11?"}],
+        },
         {
             "role": "assistant",
             "content": [
@@ -1522,7 +1546,10 @@ FOLLOW_UP_CONVERSATION_EVENTS: list[StreamEvent] = [
         "finishReason": "stop",
         "isContinued": False,
         "experimental_providerMetadata": {"openai": {"reasoningTokens": 0, "cachedPromptTokens": 0}},
-        "response": {"timestamp": "2025-01-02T12:29:13.000Z", "modelId": "gpt-4o-mini-latest"},
+        "response": {
+            "timestamp": "2025-01-02T12:29:13.000Z",
+            "modelId": "gpt-4o-mini-latest",
+        },
         "usage": {"promptTokens": 77, "completionTokens": 3, "totalTokens": 80},
     },
     {
@@ -1530,7 +1557,10 @@ FOLLOW_UP_CONVERSATION_EVENTS: list[StreamEvent] = [
         "type": "finish",
         "finishReason": "stop",
         "experimental_providerMetadata": {"openai": {"reasoningTokens": 0, "cachedPromptTokens": 0}},
-        "response": {"timestamp": "2025-01-02T12:29:13.000Z", "modelId": "gpt-4o-mini-latest"},
+        "response": {
+            "timestamp": "2025-01-02T12:29:13.000Z",
+            "modelId": "gpt-4o-mini-latest",
+        },
         "usage": {"promptTokens": 77, "completionTokens": 3, "totalTokens": 80},
     },
     ChainEventProviderCompleted(