lmnr 0.4.53.dev0__py3-none-any.whl → 0.7.26__py3-none-any.whl

lmnr/sdk/client/asynchronous/resources/evals.py

@@ -0,0 +1,266 @@
+"""Evals resource for interacting with Laminar evaluations API."""
+
+import uuid
+import warnings
+
+from typing import Any
+
+from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.types import (
+    GetDatapointsResponse,
+    InitEvaluationResponse,
+    EvaluationResultDatapoint,
+    PartialEvaluationDatapoint,
+)
+from lmnr.sdk.utils import serialize
+
+INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH = 16_000_000  # 16MB
+logger = get_default_logger(__name__)
+
+
+class AsyncEvals(BaseAsyncResource):
+    """Resource for interacting with Laminar evaluations API."""
+
+    async def init(
+        self,
+        name: str | None = None,
+        group_name: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> InitEvaluationResponse:
+        """Initialize a new evaluation.
+
+        Args:
+            name (str | None, optional): Name of the evaluation. Defaults to None.
+            group_name (str | None, optional): Group name for the evaluation. Defaults to None.
+            metadata (dict[str, Any] | None, optional): Metadata to associate with. Defaults to None.
+
+        Returns:
+            InitEvaluationResponse: The response from the initialization request.
+        """
+        response = await self._client.post(
+            self._base_url + "/v1/evals",
+            json={
+                "name": name,
+                "groupName": group_name,
+                "metadata": metadata,
+            },
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error initializing evaluation: {response.text}")
+        resp_json = response.json()
+        return InitEvaluationResponse.model_validate(resp_json)
+
+    async def create_evaluation(
+        self,
+        name: str | None = None,
+        group_name: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a new evaluation and return its ID.
+
+        Parameters:
+            name (str | None, optional): Optional name of the evaluation.
+            group_name (str | None, optional): An identifier to group evaluations.
+            metadata (dict[str, Any] | None, optional): Metadata to associate with. Defaults to None.
+
+        Returns:
+            uuid.UUID: The evaluation ID.
+        """
+        evaluation = await self.init(
+            name=name, group_name=group_name, metadata=metadata
+        )
+        return evaluation.id
+
+    async def create_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        data: Any,
+        target: Any = None,
+        metadata: dict[str, Any] | None = None,
+        index: int | None = None,
+        trace_id: uuid.UUID | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a datapoint for an evaluation.
+
+        Parameters:
+            eval_id (uuid.UUID): The evaluation ID.
+            data: The input data for the executor.
+            target: The target/expected output for evaluators.
+            metadata (dict[str, Any] | None, optional): Optional metadata.
+            index (int | None, optional): Optional index of the datapoint.
+            trace_id (uuid.UUID | None, optional): Optional trace ID.
+
+        Returns:
+            uuid.UUID: The datapoint ID.
+        """
+
+        datapoint_id = uuid.uuid4()
+
+        # Create a minimal datapoint first
+        partial_datapoint = PartialEvaluationDatapoint(
+            id=datapoint_id,
+            data=data,
+            target=target,
+            index=index or 0,
+            trace_id=trace_id or uuid.uuid4(),
+            executor_span_id=uuid.uuid4(),  # Will be updated when executor runs
+            metadata=metadata,
+        )
+
+        await self.save_datapoints(eval_id, [partial_datapoint])
+        return datapoint_id
+
+    async def save_datapoints(
+        self,
+        eval_id: uuid.UUID,
+        datapoints: list[EvaluationResultDatapoint | PartialEvaluationDatapoint],
+        group_name: str | None = None,
+    ):
+        """Save evaluation datapoints.
+
+        Args:
+            eval_id (uuid.UUID): The evaluation ID.
+            datapoints (list[EvaluationResultDatapoint | PartialEvaluationDatapoint]): The datapoints to save.
+            group_name (str | None, optional): Group name for the datapoints. Defaults to None.
+
+        Raises:
+            ValueError: If there's an error saving the datapoints.
+        """
+        length = INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+        points = [datapoint.to_dict(max_data_length=length) for datapoint in datapoints]
+        response = await self._client.post(
+            self._base_url + f"/v1/evals/{eval_id}/datapoints",
+            json={
+                "points": points,
+                "groupName": group_name,
+            },
+            headers=self._headers(),
+        )
+        if response.status_code == 413:
+            await self._retry_save_datapoints(eval_id, datapoints, group_name)
+            return
+
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error saving evaluation datapoints: [{response.status_code}] {response.text}"
+            )
+
+    async def get_datapoints(
+        self,
+        dataset_name: str,
+        offset: int,
+        limit: int,
+    ) -> GetDatapointsResponse:
+        """Get datapoints from a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            offset (int): The offset to start from.
+            limit (int): The maximum number of datapoints to return.
+
+        Returns:
+            GetDatapointsResponse: The response containing the datapoints.
+
+        Raises:
+            ValueError: If there's an error fetching the datapoints.
+        """
+        warnings.warn(
+            "Use client.datasets.pull instead",
+            DeprecationWarning,
+        )
+
+        params = {"name": dataset_name, "offset": offset, "limit": limit}
+        response = await self._client.get(
+            self._base_url + "/v1/datasets/datapoints",
+            params=params,
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            try:
+                resp_json = response.json()
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {resp_json}"
+                )
+            except Exception:
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {response.text}"
+                )
+        return GetDatapointsResponse.model_validate(response.json())
+
+    async def update_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        datapoint_id: uuid.UUID,
+        scores: dict[str, float | int],
+        executor_output: Any | None = None,
+    ) -> None:
+        """Update a datapoint with evaluation results.
+
+        Args:
+            eval_id (uuid.UUID): The evaluation ID.
+            datapoint_id (uuid.UUID): The datapoint ID.
+            executor_output (Any): The executor output.
+            scores (dict[str, float | int] | None, optional): The scores. Defaults to None.
+        """
+
+        response = await self._client.post(
+            self._base_url + f"/v1/evals/{eval_id}/datapoints/{datapoint_id}",
+            json={
+                "executorOutput": (
+                    str(serialize(executor_output))[
+                        :INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+                    ]
+                    if executor_output is not None
+                    else None
+                ),
+                "scores": scores,
+            },
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error updating evaluation datapoint: [{response.status_code}] {response.text}"
+            )
+
+    async def _retry_save_datapoints(
+        self,
+        eval_id: uuid.UUID,
+        datapoints: list[EvaluationResultDatapoint | PartialEvaluationDatapoint],
+        group_name: str | None = None,
+        initial_length: int = INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH,
+        max_retries: int = 20,
+    ):
+        retry = 0
+        length = initial_length
+        while retry < max_retries:
+            retry += 1
+            length = length // 2
+            logger.debug(
+                f"Retrying save datapoints: {retry} of {max_retries}, length: {length}"
+            )
+            if length == 0:
+                raise ValueError("Error saving evaluation datapoints")
+            points = [
+                datapoint.to_dict(max_data_length=length) for datapoint in datapoints
+            ]
+            response = await self._client.post(
+                self._base_url + f"/v1/evals/{eval_id}/datapoints",
+                json={
+                    "points": points,
+                    "groupName": group_name,
+                },
+                headers=self._headers(),
+            )
+            if response.status_code != 413:
+                break
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error saving evaluation datapoints: [{response.status_code}] {response.text}"
+            )

lmnr/sdk/client/asynchronous/resources/evaluators.py

@@ -0,0 +1,85 @@
+"""Evaluators resource for creating evaluator scores."""
+
+import uuid
+from typing import Any
+
+from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.utils import format_id
+
+
+class AsyncEvaluators(BaseAsyncResource):
+    """Resource for creating evaluator scores."""
+
+    async def score(
+        self,
+        *,
+        name: str,
+        trace_id: str | int | uuid.UUID | None = None,
+        span_id: str | int | uuid.UUID | None = None,
+        metadata: dict[str, Any] | None = None,
+        score: float,
+    ) -> None:
+        """Create a score for a span.
+
+        Args:
+            name (str): Name of the score
+            trace_id (str | int | uuid.UUID | None, optional): The trace ID to score (will be attached to root span)
+            span_id (str | int | uuid.UUID | None, optional): The span ID to score
+            metadata (dict[str, Any] | None, optional): Additional metadata. Defaults to None.
+            score (float): The score value (float)
+
+        Raises:
+            ValueError: If there's an error creating the score.
+
+        Example:
+            Score by trace ID (will attach to root span):
+
+            >>> await laminar_client.evaluators.score(
+            ...     name="quality",
+            ...     trace_id="trace-id-here",
+            ...     score=0.95,
+            ...     metadata={"model": "gpt-4"}
+            ... )
+
+            Score by span ID:
+
+            >>> await laminar_client.evaluators.score(
+            ...     name="relevance",
+            ...     span_id="span-id-here",
+            ...     score=0.87
+            ... )
+        """
+        if trace_id is not None and span_id is not None:
+            raise ValueError("Cannot provide both trace_id and span_id. Please provide only one.")
+        if trace_id is None and span_id is None:
+            raise ValueError("Either 'trace_id' or 'span_id' must be provided.")
+
+        if trace_id is not None:
+            formatted_trace_id = format_id(trace_id)
+            payload = {
+                "name": name,
+                "traceId": formatted_trace_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+        else:
+            formatted_span_id = format_id(span_id)
+            payload = {
+                "name": name,
+                "spanId": formatted_span_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+
+        response = await self._client.post(
+            self._base_url + "/v1/evaluators/score",
+            json=payload,
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error creating evaluator score: {response.text}")

lmnr/sdk/client/asynchronous/resources/tags.py

@@ -0,0 +1,83 @@
+"""Resource for tagging traces."""
+
+import json
+import uuid
+
+from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.utils import format_id
+
+logger = get_default_logger(__name__)
+
+
+class AsyncTags(BaseAsyncResource):
+    """Resource for tagging traces."""
+
+    async def tag(
+        self,
+        trace_id: str | int | uuid.UUID,
+        tags: list[str] | str,
+    ):
+        """Tag a trace with a list of tags. Note that the trace must be ended
+        before tagging it. You may want to call `Laminar.flush()` after the
+        trace that you want to tag.
+
+        Args:
+            trace_id (str | int | uuid.UUID): The trace id to tag.
+            tags (list[str] | str): The tag or list of tags to add to the trace.
+
+        Raises:
+            ValueError: If the trace id is not a valid UUID.
+
+        Returns:
+            list[dict]: The response from the server.
+
+        Example:
+        ```python
+        from lmnr import Laminar, AsyncLaminarClient, observe
+
+        Laminar.initialize()
+        client = AsyncLaminarClient()
+        trace_id = None
+
+        @observe()
+        def foo():
+            trace_id = Laminar.get_trace_id()
+            pass
+
+        # make sure `foo` is called outside a trace context
+        foo()
+
+        # or make sure the trace is ended by this point
+        Laminar.flush()
+
+        await client.tags.tag(trace_id, "my_tag")
+        ```
+        """
+        trace_tags = tags if isinstance(tags, list) else [tags]
+        formatted_trace_id = format_id(trace_id)
+
+        url = self._base_url + "/v1/tag"
+        payload = {
+            "traceId": formatted_trace_id,
+            "names": trace_tags,
+        }
+        response = await self._client.post(
+            url,
+            content=json.dumps(payload),
+            headers={
+                **self._headers(),
+            },
+        )
+
+        if response.status_code == 404:
+            logger.warning(
+                f"Trace {formatted_trace_id} not found. The trace may have not been ended yet."
+            )
+            return []
+
+        if response.status_code != 200:
+            raise ValueError(
+                f"Failed to tag trace: [{response.status_code}] {response.text}"
+            )
+        return response.json()

lmnr/sdk/client/synchronous/resources/__init__.py

@@ -0,0 +1,6 @@
+from lmnr.sdk.client.synchronous.resources.browser_events import BrowserEvents
+from lmnr.sdk.client.synchronous.resources.evals import Evals
+from lmnr.sdk.client.synchronous.resources.tags import Tags
+from lmnr.sdk.client.synchronous.resources.evaluators import Evaluators
+
+__all__ = ["Evals", "Evaluators", "BrowserEvents", "Tags"]

lmnr/sdk/client/synchronous/resources/base.py

@@ -0,0 +1,32 @@
+"""Base class for resource objects."""
+
+import httpx
+
+
+class BaseResource:
+    """Base class for all API resources."""
+
+    def __init__(self, client: httpx.Client, base_url: str, project_api_key: str):
+        """Initialize the resource.
+
+        Args:
+            client (httpx.Client): HTTP client instance
+            base_url (str): Base URL for the API
+            project_api_key (str): Project API key
+        """
+        self._client = client
+        self._base_url = base_url
+        self._project_api_key = project_api_key
+
+    def _headers(self) -> dict[str, str]:
+        """Generate request headers with authentication.
+
+        Returns:
+            dict[str, str]: Headers dictionary
+        """
+        assert self._project_api_key is not None, "Project API key is not set"
+        return {
+            "Authorization": "Bearer " + self._project_api_key,
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+        }

lmnr/sdk/client/synchronous/resources/browser_events.py

@@ -0,0 +1,40 @@
+"""Resource for sending browser events."""
+
+import gzip
+import json
+
+from lmnr.sdk.client.synchronous.resources.base import BaseResource
+
+from lmnr.version import PYTHON_VERSION, __version__
+
+
+class BrowserEvents(BaseResource):
+    """Resource for sending browser events."""
+
+    def send(
+        self,
+        session_id: str,
+        trace_id: str,
+        events: list[dict],
+    ):
+        url = self._base_url + "/v1/browser-sessions/events"
+        payload = {
+            "sessionId": session_id,
+            "traceId": trace_id,
+            "events": events,
+            "source": f"python@{PYTHON_VERSION}",
+            "sdkVersion": __version__,
+        }
+        compressed_payload = gzip.compress(json.dumps(payload).encode("utf-8"))
+        response = self._client.post(
+            url,
+            content=compressed_payload,
+            headers={
+                **self._headers(),
+                "Content-Encoding": "gzip",
+            },
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Failed to send events: [{response.status_code}] {response.text}"
+            )

lmnr/sdk/client/synchronous/resources/datasets.py

@@ -0,0 +1,131 @@
+"""Datasets resource for interacting with Laminar datasets API."""
+
+import math
+import uuid
+
+from lmnr.sdk.client.synchronous.resources.base import BaseResource
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.types import (
+    Datapoint,
+    Dataset,
+    GetDatapointsResponse,
+    PushDatapointsResponse,
+)
+from lmnr.sdk.utils import serialize
+
+logger = get_default_logger(__name__)
+
+DEFAULT_DATASET_PULL_LIMIT = 100
+DEFAULT_DATASET_PUSH_BATCH_SIZE = 100
+
+
+class Datasets(BaseResource):
+    """Resource for interacting with Laminar datasets API."""
+
+    def list_datasets(self) -> list[Dataset]:
+        """List all datasets."""
+        response = self._client.get(
+            f"{self._base_url}/v1/datasets",
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error listing datasets: [{response.status_code}] {response.text}"
+            )
+        return [Dataset.model_validate(dataset) for dataset in response.json()]
+
+    def get_dataset_by_name(self, name: str) -> list[Dataset]:
+        """Get a dataset by name."""
+        response = self._client.get(
+            f"{self._base_url}/v1/datasets",
+            params={"name": name},
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error getting dataset: [{response.status_code}] {response.text}"
+            )
+        return [Dataset.model_validate(dataset) for dataset in response.json()]
+
+    def push(
+        self,
+        points: list[Datapoint],
+        name: str | None = None,
+        id: uuid.UUID | None = None,
+        batch_size: int = DEFAULT_DATASET_PUSH_BATCH_SIZE,
+        create_dataset: bool = False,
+    ) -> PushDatapointsResponse | None:
+        """Push data to a dataset."""
+
+        if name is None and id is None:
+            raise ValueError("Either name or id must be provided")
+
+        if name is not None and id is not None:
+            raise ValueError("Only one of name or id must be provided")
+
+        if create_dataset and name is None:
+            raise ValueError("Name must be provided when creating a new dataset")
+
+        identifier = {"name": name} if name is not None else {"datasetId": id}
+
+        batch_num = 0
+        total_batches = math.ceil(len(points) / batch_size)
+        response = None
+        for i in range(0, len(points), batch_size):
+            batch_num += 1
+            logger.debug(f"Pushing batch {batch_num} of {total_batches}")
+            batch = points[i : i + batch_size]
+            response = self._client.post(
+                f"{self._base_url}/v1/datasets/datapoints",
+                json={
+                    **identifier,
+                    "datapoints": [serialize(point) for point in batch],
+                    "createDataset": create_dataset,
+                },
+                headers=self._headers(),
+            )
+
+            # 201 when creating a new dataset
+            if response.status_code not in [200, 201]:
+                raise ValueError(
+                    f"Error pushing data to dataset: [{response.status_code}] {response.text}"
+                )
+
+            response = PushDatapointsResponse.model_validate(response.json())
+        # Currently, the response only contains the dataset ID,
+        # so it's safe to return the last response only.
+        return response
+
+    def pull(
+        self,
+        name: str | None = None,
+        id: uuid.UUID | None = None,
+        # TODO: move const to one file, import in CLI
+        limit: int = DEFAULT_DATASET_PULL_LIMIT,
+        offset: int = 0,
+    ) -> GetDatapointsResponse:
+        """Pull data from a dataset."""
+
+        if name is None and id is None:
+            raise ValueError("Either name or id must be provided")
+
+        if name is not None and id is not None:
+            raise ValueError("Only one of name or id must be provided")
+
+        identifier = {"name": name} if name is not None else {"datasetId": id}
+
+        params = {
+            **identifier,
+            "offset": offset,
+            "limit": limit,
+        }
+        response = self._client.get(
+            f"{self._base_url}/v1/datasets/datapoints",
+            params=params,
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error pulling data from dataset: [{response.status_code}] {response.text}"
+            )
+        return GetDatapointsResponse.model_validate(response.json())