lmnr 0.6.10__tar.gz → 0.6.11__tar.gz

{lmnr-0.6.10 → lmnr-0.6.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lmnr
-Version: 0.6.10
+Version: 0.6.11
 Summary: Python SDK for Laminar
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.6.10 → lmnr-0.6.11}/pyproject.toml

@@ -6,7 +6,7 @@
 
 [project]
 name = "lmnr"
-version = "0.6.10"
+version = "0.6.11"
 description = "Python SDK for Laminar"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }

{lmnr-0.6.10 → lmnr-0.6.11}/src/lmnr/sdk/client/asynchronous/async_client.py

@@ -89,11 +89,11 @@ class AsyncLaminarClient:
         return self.__agent
 
     @property
-    def _evals(self) -> AsyncEvals:
+    def evals(self) -> AsyncEvals:
         """Get the Evals resource.
 
         Returns:
-            Evals: The Evals resource instance.
+            AsyncEvals: The Evals resource instance.
         """
         return self.__evals
 
@@ -144,3 +144,5 @@ class AsyncLaminarClient:
             "Content-Type": "application/json",
             "Accept": "application/json",
         }
+
+

lmnr-0.6.11/src/lmnr/sdk/client/asynchronous/resources/evals.py

@@ -0,0 +1,158 @@
+"""Evals resource for interacting with Laminar evaluations API."""
+
+from typing import Any
+import uuid
+
+from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.types import (
+    InitEvaluationResponse,
+    EvaluationResultDatapoint,
+    PartialEvaluationDatapoint,
+)
+
+
+class AsyncEvals(BaseAsyncResource):
+    """Resource for interacting with Laminar evaluations API."""
+
+    async def init(
+        self, name: str | None = None, group_name: str | None = None
+    ) -> InitEvaluationResponse:
+        """Initialize a new evaluation.
+
+        Args:
+            name (str | None, optional): Name of the evaluation. Defaults to None.
+            group_name (str | None, optional): Group name for the evaluation. Defaults to None.
+
+        Returns:
+            InitEvaluationResponse: The response from the initialization request.
+        """
+        response = await self._client.post(
+            self._base_url + "/v1/evals",
+            json={
+                "name": name,
+                "groupName": group_name,
+            },
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error initializing evaluation: {response.text}")
+        resp_json = response.json()
+        return InitEvaluationResponse.model_validate(resp_json)
+
+    async def create_evaluation(
+        self,
+        name: str | None = None,
+        group_name: str | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a new evaluation and return its ID.
+        
+        Parameters:
+            name (str | None, optional): Optional name of the evaluation.
+            group_name (str | None, optional): An identifier to group evaluations.
+        
+        Returns:
+            uuid.UUID: The evaluation ID.
+        """
+        evaluation = await self.init(name=name, group_name=group_name)
+        return evaluation.id
+
+    async def create_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        data: Any,
+        target: Any = None,
+        metadata: dict[str, Any] | None = None,
+        index: int | None = None,
+        trace_id: uuid.UUID | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a datapoint for an evaluation.
+        
+        Parameters:
+            eval_id (uuid.UUID): The evaluation ID.
+            data: The input data for the executor.
+            target: The target/expected output for evaluators.
+            metadata (dict[str, Any] | None, optional): Optional metadata.
+            index (int | None, optional): Optional index of the datapoint.
+            trace_id (uuid.UUID | None, optional): Optional trace ID.
+        
+        Returns:
+            uuid.UUID: The datapoint ID.
+        """
+        
+        datapoint_id = uuid.uuid4()
+        
+        # Create a minimal datapoint first
+        partial_datapoint = PartialEvaluationDatapoint(
+            id=datapoint_id,
+            data=data,
+            target=target,
+            index=index or 0,
+            trace_id=trace_id or uuid.uuid4(),
+            executor_span_id=uuid.uuid4(),  # Will be updated when executor runs
+            metadata=metadata,
+        )
+        
+        await self.save_datapoints(eval_id, [partial_datapoint])
+        return datapoint_id
+
+    async def save_datapoints(
+        self,
+        eval_id: uuid.UUID,
+        datapoints: list[EvaluationResultDatapoint | PartialEvaluationDatapoint],
+        group_name: str | None = None,
+    ):
+        """Save evaluation datapoints.
+
+        Args:
+            eval_id (uuid.UUID): The evaluation ID.
+            datapoints (list[EvaluationResultDatapoint | PartialEvaluationDatapoint]): The datapoints to save.
+            group_name (str | None, optional): Group name for the datapoints. Defaults to None.
+
+        Raises:
+            ValueError: If there's an error saving the datapoints.
+        """
+        response = await self._client.post(
+            self._base_url + f"/v1/evals/{eval_id}/datapoints",
+            json={
+                "points": [datapoint.to_dict() for datapoint in datapoints],
+                "groupName": group_name,
+            },
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            raise ValueError(f"Error saving evaluation datapoints: {response.text}")
+        
+    
+    async def update_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        datapoint_id: uuid.UUID,
+        scores: dict[str, float | int],
+        executor_output: Any | None = None,
+    ) -> None:
+        """Update a datapoint with evaluation results.
+
+        Args:
+            eval_id (uuid.UUID): The evaluation ID.
+            datapoint_id (uuid.UUID): The datapoint ID.
+            executor_output (Any): The executor output.
+            scores (dict[str, float | int] | None, optional): The scores. Defaults to None.
+        """
+        
+        response = await self._client.post(
+            self._base_url + f"/v1/evals/{eval_id}/datapoints/{datapoint_id}",
+            json={
+                "executorOutput": executor_output,
+                "scores": scores,
+            },
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            raise ValueError(f"Error updating evaluation datapoint: {response.text}")
+        
+    

{lmnr-0.6.10 → lmnr-0.6.11}/src/lmnr/sdk/client/synchronous/resources/evals.py

@@ -2,6 +2,7 @@
 
 import uuid
 import urllib.parse
+from typing import Any
 
 from lmnr.sdk.client.synchronous.resources.base import BaseResource
 from lmnr.sdk.types import (
@@ -42,6 +43,64 @@ class Evals(BaseResource):
         resp_json = response.json()
         return InitEvaluationResponse.model_validate(resp_json)
 
+    def create_evaluation(
+        self,
+        name: str | None = None,
+        group_name: str | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a new evaluation and return its ID.
+        
+        Parameters:
+            name (str | None, optional): Optional name of the evaluation.
+            group_name (str | None, optional): An identifier to group evaluations.
+        
+        Returns:
+            uuid.UUID: The evaluation ID.
+        """
+        evaluation = self.init(name=name, group_name=group_name)
+        return evaluation.id
+
+    def create_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        data: Any,
+        target: Any = None,
+        metadata: dict[str, Any] | None = None,
+        index: int | None = None,
+        trace_id: uuid.UUID | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a datapoint for an evaluation.
+        
+        Parameters:
+            eval_id (uuid.UUID): The evaluation ID.
+            data: The input data for the executor.
+            target: The target/expected output for evaluators.
+            metadata (dict[str, Any] | None, optional): Optional metadata.
+            index (int | None, optional): Optional index of the datapoint.
+            trace_id (uuid.UUID | None, optional): Optional trace ID.
+        
+        Returns:
+            uuid.UUID: The datapoint ID.
+        """
+        
+        datapoint_id = uuid.uuid4()
+        
+        # Create a minimal datapoint first
+        partial_datapoint = PartialEvaluationDatapoint(
+            id=datapoint_id,
+            data=data,
+            target=target,
+            index=index or 0,
+            trace_id=trace_id or uuid.uuid4(),
+            executor_span_id=uuid.uuid4(),  # Will be updated when executor runs
+            metadata=metadata,
+        )
+        
+        self.save_datapoints(eval_id, [partial_datapoint])
+        return datapoint_id
+
     def save_datapoints(
         self,
         eval_id: uuid.UUID,
@@ -69,6 +128,34 @@ class Evals(BaseResource):
         if response.status_code != 200:
             raise ValueError(f"Error saving evaluation datapoints: {response.text}")
 
+    def update_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        datapoint_id: uuid.UUID,
+        scores: dict[str, float | int],
+        executor_output: Any | None = None,
+    ) -> None:
+        """Update a datapoint with evaluation results.
+
+        Args:
+            eval_id (uuid.UUID): The evaluation ID.
+            datapoint_id (uuid.UUID): The datapoint ID.
+            executor_output (Any): The executor output.
+            scores (dict[str, float | int] | None, optional): The scores. Defaults to None.
+        """
+        
+        response = self._client.post(
+            self._base_url + f"/v1/evals/{eval_id}/datapoints/{datapoint_id}",
+            json={
+                "executorOutput": executor_output,
+                "scores": scores,
+            },
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            raise ValueError(f"Error updating evaluation datapoint: {response.text}")
+
     def get_datapoints(
         self,
         dataset_name: str,

{lmnr-0.6.10 → lmnr-0.6.11}/src/lmnr/sdk/client/synchronous/sync_client.py

@@ -89,7 +89,7 @@ class LaminarClient:
         return self.__agent
 
     @property
-    def _evals(self) -> Evals:
+    def evals(self) -> Evals:
         """Get the Evals resource.
 
         Returns:
@@ -155,3 +155,5 @@ class LaminarClient:
             "Content-Type": "application/json",
             "Accept": "application/json",
         }
+
+

{lmnr-0.6.10 → lmnr-0.6.11}/src/lmnr/sdk/datasets.py

@@ -38,7 +38,7 @@ class LaminarDataset(EvaluationDataset):
             f"dataset {self.name}. Fetching batch from {self._offset} to "
             + f"{self._offset + self._fetch_size}"
         )
-        resp = self.client._evals.get_datapoints(
+        resp = self.client.evals.get_datapoints(
             self.name, self._offset, self._fetch_size
         )
         self._fetched_items += resp.items

{lmnr-0.6.10 → lmnr-0.6.11}/src/lmnr/sdk/evaluations.py

@@ -241,7 +241,7 @@ class Evaluation:
             )
         self.reporter.start(len(self.data))
         try:
-            evaluation = await self.client._evals.init(
+            evaluation = await self.client.evals.init(
                 name=self.name, group_name=self.group_name
             )
             result_datapoints = await self._evaluate_in_batches(evaluation.id)
@@ -326,7 +326,7 @@ class Evaluation:
                     metadata=datapoint.metadata,
                 )
                 # First, create datapoint with trace_id so that we can show the dp in the UI
-                await self.client._evals.save_datapoints(
+                await self.client.evals.save_datapoints(
                     eval_id, [partial_datapoint], self.group_name
                 )
                 executor_span.set_attribute(SPAN_TYPE, SpanType.EXECUTOR.value)
@@ -384,7 +384,7 @@ class Evaluation:
 
         # Create background upload task without awaiting it
         upload_task = asyncio.create_task(
-            self.client._evals.save_datapoints(eval_id, [datapoint], self.group_name)
+            self.client.evals.save_datapoints(eval_id, [datapoint], self.group_name)
         )
         self.upload_tasks.append(upload_task)
 

{lmnr-0.6.10 → lmnr-0.6.11}/src/lmnr/version.py

@@ -3,7 +3,7 @@ import httpx
 from packaging import version
 
 
-__version__ = "0.6.10"
+__version__ = "0.6.11"
 PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
 
 

lmnr-0.6.10/src/lmnr/sdk/client/asynchronous/resources/evals.py

@@ -1,68 +0,0 @@
-"""Evals resource for interacting with Laminar evaluations API."""
-
-import uuid
-
-from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
-from lmnr.sdk.types import (
-    InitEvaluationResponse,
-    EvaluationResultDatapoint,
-    PartialEvaluationDatapoint,
-)
-
-
-class AsyncEvals(BaseAsyncResource):
-    """Resource for interacting with Laminar evaluations API."""
-
-    async def init(
-        self, name: str | None = None, group_name: str | None = None
-    ) -> InitEvaluationResponse:
-        """Initialize a new evaluation.
-
-        Args:
-            name (str | None, optional): Name of the evaluation. Defaults to None.
-            group_name (str | None, optional): Group name for the evaluation. Defaults to None.
-
-        Returns:
-            InitEvaluationResponse: The response from the initialization request.
-        """
-        response = await self._client.post(
-            self._base_url + "/v1/evals",
-            json={
-                "name": name,
-                "groupName": group_name,
-            },
-            headers=self._headers(),
-        )
-        if response.status_code != 200:
-            if response.status_code == 401:
-                raise ValueError("Unauthorized. Please check your project API key.")
-            raise ValueError(f"Error initializing evaluation: {response.text}")
-        resp_json = response.json()
-        return InitEvaluationResponse.model_validate(resp_json)
-
-    async def save_datapoints(
-        self,
-        eval_id: uuid.UUID,
-        datapoints: list[EvaluationResultDatapoint | PartialEvaluationDatapoint],
-        group_name: str | None = None,
-    ):
-        """Save evaluation datapoints.
-
-        Args:
-            eval_id (uuid.UUID): The evaluation ID.
-            datapoints (list[EvaluationResultDatapoint | PartialEvaluationDatapoint]): The datapoints to save.
-            group_name (str | None, optional): Group name for the datapoints. Defaults to None.
-
-        Raises:
-            ValueError: If there's an error saving the datapoints.
-        """
-        response = await self._client.post(
-            self._base_url + f"/v1/evals/{eval_id}/datapoints",
-            json={
-                "points": [datapoint.to_dict() for datapoint in datapoints],
-                "groupName": group_name,
-            },
-            headers=self._headers(),
-        )
-        if response.status_code != 200:
-            raise ValueError(f"Error saving evaluation datapoints: {response.text}")