lmnr 0.6.16__py3-none-any.whl → 0.7.26__py3-none-any.whl

lmnr/sdk/client/asynchronous/async_client.py

@@ -8,11 +8,12 @@ from typing import TypeVar
 from types import TracebackType
 
 from lmnr.sdk.client.asynchronous.resources import (
-    AsyncAgent,
     AsyncBrowserEvents,
     AsyncEvals,
     AsyncTags,
+    AsyncEvaluators,
 )
+from lmnr.sdk.client.asynchronous.resources.datasets import AsyncDatasets
 from lmnr.sdk.utils import from_env
 
 _T = TypeVar("_T", bound="AsyncLaminarClient")
@@ -65,28 +66,42 @@ class AsyncLaminarClient:
         self.__client = httpx.AsyncClient(
             headers=self._headers(),
             timeout=timeout,
+            # Context: If the server responds with a 413, the connection becomes
+            # poisoned and freezes on subsequent requests, and there is no way
+            # to recover or recycle such connection.
+            # Setting max_keepalive_connections to 0 will resolve this, but is
+            # less efficient, as it will create a new connection
+            # (not client, so still better) for each request.
+            #
+            # Note: from my experiments with a simple python server, forcing the
+            # server to read/consume the request payload from the socket seems
+            # to resolve this, but I haven't figured out how to do that in our
+            # real actix-web backend server and whether it makes sense to do so.
+            #
+            # TODO: investigate if there are better ways to fix this rather than
+            # setting keepalive_expiry to 0. Other alternative: migrate to
+            # requests + aiohttp.
+            #
+            # limits=httpx.Limits(
+            #     max_keepalive_connections=0,
+            #     keepalive_expiry=0,
+            # ),
         )
 
         # Initialize resource objects
-        self.__agent = AsyncAgent(
+        self.__evals = AsyncEvals(
             self.__client, self.__base_url, self.__project_api_key
         )
-        self.__evals = AsyncEvals(
+        self.__evaluators = AsyncEvaluators(
             self.__client, self.__base_url, self.__project_api_key
         )
         self.__browser_events = AsyncBrowserEvents(
             self.__client, self.__base_url, self.__project_api_key
         )
         self.__tags = AsyncTags(self.__client, self.__base_url, self.__project_api_key)
-
-    @property
-    def agent(self) -> AsyncAgent:
-        """Get the Agent resource.
-
-        Returns:
-            Agent: The Agent resource instance.
-        """
-        return self.__agent
+        self.__datasets = AsyncDatasets(
+            self.__client, self.__base_url, self.__project_api_key
+        )
 
     @property
     def evals(self) -> AsyncEvals:
@@ -102,7 +117,7 @@ class AsyncLaminarClient:
         """Get the BrowserEvents resource.
 
         Returns:
-            BrowserEvents: The BrowserEvents resource instance.
+            AsyncBrowserEvents: The BrowserEvents resource instance.
         """
         return self.__browser_events
 
@@ -115,6 +130,25 @@ class AsyncLaminarClient:
         """
         return self.__tags
 
+    @property
+    def evaluators(self) -> AsyncEvaluators:
+        """Get the Evaluators resource.
+
+        Returns:
+            AsyncEvaluators: The Evaluators resource instance.
+        """
+        return self.__evaluators
+
+    @property
+    def datasets(self) -> AsyncDatasets:
+        """Get the Datasets resource.
+
+        Returns:
+            AsyncDatasets: The Datasets resource instance.
+        """
+        return self.__datasets
+
+    @property
     def is_closed(self) -> bool:
         return self.__client.is_closed
 
@@ -144,5 +178,3 @@ class AsyncLaminarClient:
             "Content-Type": "application/json",
             "Accept": "application/json",
         }
-
-

lmnr/sdk/client/asynchronous/resources/__init__.py

@@ -1,11 +1,6 @@
-from lmnr.sdk.client.asynchronous.resources.agent import AsyncAgent
 from lmnr.sdk.client.asynchronous.resources.browser_events import AsyncBrowserEvents
 from lmnr.sdk.client.asynchronous.resources.evals import AsyncEvals
 from lmnr.sdk.client.asynchronous.resources.tags import AsyncTags
+from lmnr.sdk.client.asynchronous.resources.evaluators import AsyncEvaluators
 
-__all__ = [
-    "AsyncAgent",
-    "AsyncEvals",
-    "AsyncBrowserEvents",
-    "AsyncTags",
-]
+__all__ = ["AsyncEvals", "AsyncBrowserEvents", "AsyncTags", "AsyncEvaluators"]

lmnr/sdk/client/asynchronous/resources/browser_events.py

@@ -25,6 +25,7 @@ class AsyncBrowserEvents(BaseAsyncResource):
             "source": f"python@{PYTHON_VERSION}",
             "sdkVersion": __version__,
         }
+
         compressed_payload = gzip.compress(json.dumps(payload).encode("utf-8"))
         response = await self._client.post(
             url,

lmnr/sdk/client/asynchronous/resources/datasets.py

@@ -0,0 +1,131 @@
+"""Datasets resource for interacting with Laminar datasets API."""
+
+import math
+import uuid
+
+from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.types import (
+    Datapoint,
+    Dataset,
+    GetDatapointsResponse,
+    PushDatapointsResponse,
+)
+from lmnr.sdk.utils import serialize
+
+logger = get_default_logger(__name__)
+
+DEFAULT_DATASET_PULL_LIMIT = 100
+DEFAULT_DATASET_PUSH_BATCH_SIZE = 100
+
+
+class AsyncDatasets(BaseAsyncResource):
+    """Resource for interacting with Laminar datasets API."""
+
+    async def list_datasets(self) -> list[Dataset]:
+        """List all datasets."""
+        response = await self._client.get(
+            f"{self._base_url}/v1/datasets",
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error listing datasets: [{response.status_code}] {response.text}"
+            )
+        return [Dataset.model_validate(dataset) for dataset in response.json()]
+
+    async def get_dataset_by_name(self, name: str) -> list[Dataset]:
+        """Get a dataset by name."""
+        response = await self._client.get(
+            f"{self._base_url}/v1/datasets",
+            params={"name": name},
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error getting dataset: [{response.status_code}] {response.text}"
+            )
+        return [Dataset.model_validate(dataset) for dataset in response.json()]
+
+    async def push(
+        self,
+        points: list[Datapoint],
+        name: str | None = None,
+        id: uuid.UUID | None = None,
+        batch_size: int = DEFAULT_DATASET_PUSH_BATCH_SIZE,
+        create_dataset: bool = False,
+    ) -> PushDatapointsResponse | None:
+        """Push data to a dataset."""
+
+        if name is None and id is None:
+            raise ValueError("Either name or id must be provided")
+
+        if name is not None and id is not None:
+            raise ValueError("Only one of name or id must be provided")
+
+        if create_dataset and name is None:
+            raise ValueError("Name must be provided when creating a new dataset")
+
+        identifier = {"name": name} if name is not None else {"datasetId": id}
+
+        batch_num = 0
+        total_batches = math.ceil(len(points) / batch_size)
+        response = None
+        for i in range(0, len(points), batch_size):
+            batch_num += 1
+            logger.debug(f"Pushing batch {batch_num} of {total_batches}")
+            batch = points[i : i + batch_size]
+            response = await self._client.post(
+                f"{self._base_url}/v1/datasets/datapoints",
+                json={
+                    **identifier,
+                    "datapoints": [serialize(point) for point in batch],
+                    "createDataset": create_dataset,
+                },
+                headers=self._headers(),
+            )
+
+            # 201 when creating a new dataset
+            if response.status_code not in [200, 201]:
+                raise ValueError(
+                    f"Error pushing data to dataset: [{response.status_code}] {response.text}"
+                )
+
+            response = PushDatapointsResponse.model_validate(response.json())
+        # Currently, the response only contains the dataset ID,
+        # so it's safe to return the last response only.
+        return response
+
+    async def pull(
+        self,
+        name: str | None = None,
+        id: uuid.UUID | None = None,
+        # TODO: move const to one file, import in CLI
+        limit: int = DEFAULT_DATASET_PULL_LIMIT,
+        offset: int = 0,
+    ) -> GetDatapointsResponse:
+        """Pull data from a dataset."""
+
+        if name is None and id is None:
+            raise ValueError("Either name or id must be provided")
+
+        if name is not None and id is not None:
+            raise ValueError("Only one of name or id must be provided")
+
+        identifier = {"name": name} if name is not None else {"datasetId": id}
+
+        params = {
+            **identifier,
+            "offset": offset,
+            "limit": limit,
+        }
+        response = await self._client.get(
+            f"{self._base_url}/v1/datasets/datapoints",
+            params=params,
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error pulling data from dataset: [{response.status_code}] {response.text}"
+            )
+        return GetDatapointsResponse.model_validate(response.json())

lmnr/sdk/client/asynchronous/resources/evals.py

@@ -1,21 +1,32 @@
 """Evals resource for interacting with Laminar evaluations API."""
 
-from typing import Any
 import uuid
+import warnings
+
+from typing import Any
 
 from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.log import get_default_logger
 from lmnr.sdk.types import (
+    GetDatapointsResponse,
     InitEvaluationResponse,
     EvaluationResultDatapoint,
     PartialEvaluationDatapoint,
 )
+from lmnr.sdk.utils import serialize
+
+INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH = 16_000_000  # 16MB
+logger = get_default_logger(__name__)
 
 
 class AsyncEvals(BaseAsyncResource):
     """Resource for interacting with Laminar evaluations API."""
 
     async def init(
-        self, name: str | None = None, group_name: str | None = None, metadata: dict[str, Any] | None = None
+        self,
+        name: str | None = None,
+        group_name: str | None = None,
+        metadata: dict[str, Any] | None = None,
     ) -> InitEvaluationResponse:
         """Initialize a new evaluation.
 
@@ -51,7 +62,7 @@ class AsyncEvals(BaseAsyncResource):
     ) -> uuid.UUID:
         """
         Create a new evaluation and return its ID.
-        
+
         Parameters:
             name (str | None, optional): Optional name of the evaluation.
             group_name (str | None, optional): An identifier to group evaluations.
@@ -60,7 +71,9 @@ class AsyncEvals(BaseAsyncResource):
         Returns:
             uuid.UUID: The evaluation ID.
         """
-        evaluation = await self.init(name=name, group_name=group_name, metadata=metadata)
+        evaluation = await self.init(
+            name=name, group_name=group_name, metadata=metadata
+        )
         return evaluation.id
 
     async def create_datapoint(
@@ -74,7 +87,7 @@ class AsyncEvals(BaseAsyncResource):
     ) -> uuid.UUID:
         """
         Create a datapoint for an evaluation.
-        
+
         Parameters:
             eval_id (uuid.UUID): The evaluation ID.
             data: The input data for the executor.
@@ -82,13 +95,13 @@ class AsyncEvals(BaseAsyncResource):
             metadata (dict[str, Any] | None, optional): Optional metadata.
             index (int | None, optional): Optional index of the datapoint.
             trace_id (uuid.UUID | None, optional): Optional trace ID.
-        
+
         Returns:
             uuid.UUID: The datapoint ID.
         """
-        
+
         datapoint_id = uuid.uuid4()
-        
+
         # Create a minimal datapoint first
         partial_datapoint = PartialEvaluationDatapoint(
             id=datapoint_id,
@@ -99,7 +112,7 @@ class AsyncEvals(BaseAsyncResource):
             executor_span_id=uuid.uuid4(),  # Will be updated when executor runs
             metadata=metadata,
         )
-        
+
         await self.save_datapoints(eval_id, [partial_datapoint])
         return datapoint_id
 
@@ -119,18 +132,67 @@ class AsyncEvals(BaseAsyncResource):
         Raises:
             ValueError: If there's an error saving the datapoints.
         """
+        length = INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+        points = [datapoint.to_dict(max_data_length=length) for datapoint in datapoints]
         response = await self._client.post(
             self._base_url + f"/v1/evals/{eval_id}/datapoints",
             json={
-                "points": [datapoint.to_dict() for datapoint in datapoints],
+                "points": points,
                 "groupName": group_name,
             },
             headers=self._headers(),
         )
+        if response.status_code == 413:
+            await self._retry_save_datapoints(eval_id, datapoints, group_name)
+            return
+
         if response.status_code != 200:
-            raise ValueError(f"Error saving evaluation datapoints: {response.text}")
-        
-    
+            raise ValueError(
+                f"Error saving evaluation datapoints: [{response.status_code}] {response.text}"
+            )
+
+    async def get_datapoints(
+        self,
+        dataset_name: str,
+        offset: int,
+        limit: int,
+    ) -> GetDatapointsResponse:
+        """Get datapoints from a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            offset (int): The offset to start from.
+            limit (int): The maximum number of datapoints to return.
+
+        Returns:
+            GetDatapointsResponse: The response containing the datapoints.
+
+        Raises:
+            ValueError: If there's an error fetching the datapoints.
+        """
+        warnings.warn(
+            "Use client.datasets.pull instead",
+            DeprecationWarning,
+        )
+
+        params = {"name": dataset_name, "offset": offset, "limit": limit}
+        response = await self._client.get(
+            self._base_url + "/v1/datasets/datapoints",
+            params=params,
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            try:
+                resp_json = response.json()
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {resp_json}"
+                )
+            except Exception:
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {response.text}"
+                )
+        return GetDatapointsResponse.model_validate(response.json())
+
     async def update_datapoint(
         self,
         eval_id: uuid.UUID,
@@ -146,17 +208,59 @@ class AsyncEvals(BaseAsyncResource):
             executor_output (Any): The executor output.
             scores (dict[str, float | int] | None, optional): The scores. Defaults to None.
         """
-        
+
         response = await self._client.post(
             self._base_url + f"/v1/evals/{eval_id}/datapoints/{datapoint_id}",
             json={
-                "executorOutput": executor_output,
+                "executorOutput": (
+                    str(serialize(executor_output))[
+                        :INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+                    ]
+                    if executor_output is not None
+                    else None
+                ),
                 "scores": scores,
             },
             headers=self._headers(),
         )
 
         if response.status_code != 200:
-            raise ValueError(f"Error updating evaluation datapoint: {response.text}")
-        
-    
+            raise ValueError(
+                f"Error updating evaluation datapoint: [{response.status_code}] {response.text}"
+            )
+
+    async def _retry_save_datapoints(
+        self,
+        eval_id: uuid.UUID,
+        datapoints: list[EvaluationResultDatapoint | PartialEvaluationDatapoint],
+        group_name: str | None = None,
+        initial_length: int = INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH,
+        max_retries: int = 20,
+    ):
+        retry = 0
+        length = initial_length
+        while retry < max_retries:
+            retry += 1
+            length = length // 2
+            logger.debug(
+                f"Retrying save datapoints: {retry} of {max_retries}, length: {length}"
+            )
+            if length == 0:
+                raise ValueError("Error saving evaluation datapoints")
+            points = [
+                datapoint.to_dict(max_data_length=length) for datapoint in datapoints
+            ]
+            response = await self._client.post(
+                self._base_url + f"/v1/evals/{eval_id}/datapoints",
+                json={
+                    "points": points,
+                    "groupName": group_name,
+                },
+                headers=self._headers(),
+            )
+            if response.status_code != 413:
+                break
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error saving evaluation datapoints: [{response.status_code}] {response.text}"
+            )

lmnr/sdk/client/asynchronous/resources/evaluators.py

@@ -0,0 +1,85 @@
+"""Evaluators resource for creating evaluator scores."""
+
+import uuid
+from typing import Any
+
+from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.utils import format_id
+
+
+class AsyncEvaluators(BaseAsyncResource):
+    """Resource for creating evaluator scores."""
+
+    async def score(
+        self,
+        *,
+        name: str,
+        trace_id: str | int | uuid.UUID | None = None,
+        span_id: str | int | uuid.UUID | None = None,
+        metadata: dict[str, Any] | None = None,
+        score: float,
+    ) -> None:
+        """Create a score for a span.
+
+        Args:
+            name (str): Name of the score
+            trace_id (str | int | uuid.UUID | None, optional): The trace ID to score (will be attached to root span)
+            span_id (str | int | uuid.UUID | None, optional): The span ID to score
+            metadata (dict[str, Any] | None, optional): Additional metadata. Defaults to None.
+            score (float): The score value (float)
+
+        Raises:
+            ValueError: If there's an error creating the score.
+
+        Example:
+            Score by trace ID (will attach to root span):
+
+            >>> await laminar_client.evaluators.score(
+            ...     name="quality",
+            ...     trace_id="trace-id-here",
+            ...     score=0.95,
+            ...     metadata={"model": "gpt-4"}
+            ... )
+
+            Score by span ID:
+
+            >>> await laminar_client.evaluators.score(
+            ...     name="relevance",
+            ...     span_id="span-id-here",
+            ...     score=0.87
+            ... )
+        """
+        if trace_id is not None and span_id is not None:
+            raise ValueError("Cannot provide both trace_id and span_id. Please provide only one.")
+        if trace_id is None and span_id is None:
+            raise ValueError("Either 'trace_id' or 'span_id' must be provided.")
+
+        if trace_id is not None:
+            formatted_trace_id = format_id(trace_id)
+            payload = {
+                "name": name,
+                "traceId": formatted_trace_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+        else:
+            formatted_span_id = format_id(span_id)
+            payload = {
+                "name": name,
+                "spanId": formatted_span_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+
+        response = await self._client.post(
+            self._base_url + "/v1/evaluators/score",
+            json=payload,
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error creating evaluator score: {response.text}")

lmnr/sdk/client/asynchronous/resources/tags.py

@@ -5,6 +5,7 @@ import uuid
 
 from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
 from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.utils import format_id
 
 logger = get_default_logger(__name__)
 
@@ -54,18 +55,11 @@ class AsyncTags(BaseAsyncResource):
         ```
         """
         trace_tags = tags if isinstance(tags, list) else [tags]
-        if isinstance(trace_id, uuid.UUID):
-            trace_id = str(trace_id)
-        elif isinstance(trace_id, int):
-            trace_id = str(uuid.UUID(int=trace_id))
-        elif isinstance(trace_id, str):
-            uuid.UUID(trace_id)  # Will raise ValueError if invalid
-        else:
-            raise ValueError(f"Invalid trace id: {trace_id}")
+        formatted_trace_id = format_id(trace_id)
 
         url = self._base_url + "/v1/tag"
         payload = {
-            "traceId": trace_id,
+            "traceId": formatted_trace_id,
             "names": trace_tags,
         }
         response = await self._client.post(
@@ -78,7 +72,7 @@ class AsyncTags(BaseAsyncResource):
 
         if response.status_code == 404:
             logger.warning(
-                f"Trace {trace_id} not found. The trace may have not been ended yet."
+                f"Trace {formatted_trace_id} not found. The trace may have not been ended yet."
             )
             return []
 

lmnr/sdk/client/synchronous/resources/__init__.py

@@ -1,6 +1,6 @@
-from lmnr.sdk.client.synchronous.resources.agent import Agent
 from lmnr.sdk.client.synchronous.resources.browser_events import BrowserEvents
 from lmnr.sdk.client.synchronous.resources.evals import Evals
 from lmnr.sdk.client.synchronous.resources.tags import Tags
+from lmnr.sdk.client.synchronous.resources.evaluators import Evaluators
 
-__all__ = ["Agent", "Evals", "BrowserEvents", "Tags"]
+__all__ = ["Evals", "Evaluators", "BrowserEvents", "Tags"]