lmnr 0.4.53.dev0__py3-none-any.whl → 0.7.26__py3-none-any.whl

lmnr/sdk/client/synchronous/resources/evals.py

@@ -0,0 +1,263 @@
+"""Evals resource for interacting with Laminar evaluations API."""
+
+import uuid
+import warnings
+
+from typing import Any
+
+from lmnr.sdk.client.synchronous.resources.base import BaseResource
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.types import (
+    GetDatapointsResponse,
+    EvaluationResultDatapoint,
+    InitEvaluationResponse,
+    PartialEvaluationDatapoint,
+)
+from lmnr.sdk.utils import serialize
+
+INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH = 16_000_000  # 16MB
+logger = get_default_logger(__name__)
+
+
+class Evals(BaseResource):
+    """Resource for interacting with Laminar evaluations API."""
+
+    def init(
+        self,
+        name: str | None = None,
+        group_name: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> InitEvaluationResponse:
+        """Initialize a new evaluation.
+
+        Args:
+            name (str | None, optional): Name of the evaluation. Defaults to None.
+            group_name (str | None, optional): Group name for the evaluation. Defaults to None.
+            metadata (dict[str, Any] | None, optional): Metadata to associate with. Defaults to None.
+
+        Returns:
+            InitEvaluationResponse: The response from the initialization request.
+        """
+        response = self._client.post(
+            self._base_url + "/v1/evals",
+            json={
+                "name": name,
+                "groupName": group_name,
+                "metadata": metadata,
+            },
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error initializing evaluation: {response.text}")
+        resp_json = response.json()
+        return InitEvaluationResponse.model_validate(resp_json)
+
+    def create_evaluation(
+        self,
+        name: str | None = None,
+        group_name: str | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a new evaluation and return its ID.
+
+        Parameters:
+            name (str | None, optional): Optional name of the evaluation.
+            group_name (str | None, optional): An identifier to group evaluations.
+            metadata (dict[str, Any] | None, optional): Metadata to associate with. Defaults to None.
+
+        Returns:
+            uuid.UUID: The evaluation ID.
+        """
+        evaluation = self.init(name=name, group_name=group_name, metadata=metadata)
+        return evaluation.id
+
+    def create_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        data: Any,
+        target: Any = None,
+        metadata: dict[str, Any] | None = None,
+        index: int | None = None,
+        trace_id: uuid.UUID | None = None,
+    ) -> uuid.UUID:
+        """
+        Create a datapoint for an evaluation.
+
+        Parameters:
+            eval_id (uuid.UUID): The evaluation ID.
+            data: The input data for the executor.
+            target: The target/expected output for evaluators.
+            metadata (dict[str, Any] | None, optional): Optional metadata.
+            index (int | None, optional): Optional index of the datapoint.
+            trace_id (uuid.UUID | None, optional): Optional trace ID.
+
+        Returns:
+            uuid.UUID: The datapoint ID.
+        """
+
+        datapoint_id = uuid.uuid4()
+
+        # Create a minimal datapoint first
+        partial_datapoint = PartialEvaluationDatapoint(
+            id=datapoint_id,
+            data=data,
+            target=target,
+            index=index or 0,
+            trace_id=trace_id or uuid.uuid4(),
+            executor_span_id=uuid.uuid4(),  # Will be updated when executor runs
+            metadata=metadata,
+        )
+
+        self.save_datapoints(eval_id, [partial_datapoint])
+        return datapoint_id
+
+    def save_datapoints(
+        self,
+        eval_id: uuid.UUID,
+        datapoints: list[EvaluationResultDatapoint | PartialEvaluationDatapoint],
+        group_name: str | None = None,
+    ):
+        """Save evaluation datapoints.
+
+        Args:
+            eval_id (uuid.UUID): The evaluation ID.
+            datapoints (list[EvaluationResultDatapoint | PartialEvaluationDatapoint]): The datapoints to save.
+            group_name (str | None, optional): Group name for the datapoints. Defaults to None.
+
+        Raises:
+            ValueError: If there's an error saving the datapoints.
+        """
+        length = INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+        points = [datapoint.to_dict(max_data_length=length) for datapoint in datapoints]
+        response = self._client.post(
+            self._base_url + f"/v1/evals/{eval_id}/datapoints",
+            json={
+                "points": points,
+                "groupName": group_name,
+            },
+            headers=self._headers(),
+        )
+        if response.status_code == 413:
+            self._retry_save_datapoints(eval_id, datapoints, group_name)
+            return
+
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error saving evaluation datapoints: [{response.status_code}] {response.text}"
+            )
+
+    def update_datapoint(
+        self,
+        eval_id: uuid.UUID,
+        datapoint_id: uuid.UUID,
+        scores: dict[str, float | int],
+        executor_output: Any | None = None,
+    ) -> None:
+        """Update a datapoint with evaluation results.
+
+        Args:
+            eval_id (uuid.UUID): The evaluation ID.
+            datapoint_id (uuid.UUID): The datapoint ID.
+            executor_output (Any): The executor output.
+            scores (dict[str, float | int] | None, optional): The scores. Defaults to None.
+        """
+
+        response = self._client.post(
+            self._base_url + f"/v1/evals/{eval_id}/datapoints/{datapoint_id}",
+            json={
+                "executorOutput": (
+                    str(serialize(executor_output))[
+                        :INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH
+                    ]
+                    if executor_output is not None
+                    else None
+                ),
+                "scores": scores,
+            },
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            raise ValueError(f"Error updating evaluation datapoint: {response.text}")
+
+    def get_datapoints(
+        self,
+        dataset_name: str,
+        offset: int,
+        limit: int,
+    ) -> GetDatapointsResponse:
+        """Get datapoints from a dataset.
+
+        Args:
+            dataset_name (str): The name of the dataset.
+            offset (int): The offset to start from.
+            limit (int): The maximum number of datapoints to return.
+
+        Returns:
+            GetDatapointsResponse: The response containing the datapoints.
+
+        Raises:
+            ValueError: If there's an error fetching the datapoints.
+        """
+
+        warnings.warn(
+            "Use client.datasets.pull instead",
+            DeprecationWarning,
+        )
+
+        params = {"name": dataset_name, "offset": offset, "limit": limit}
+        response = self._client.get(
+            self._base_url + "/v1/datasets/datapoints",
+            params=params,
+            headers=self._headers(),
+        )
+        if response.status_code != 200:
+            try:
+                resp_json = response.json()
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {resp_json}"
+                )
+            except Exception:
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {response.text}"
+                )
+        return GetDatapointsResponse.model_validate(response.json())
+
+    def _retry_save_datapoints(
+        self,
+        eval_id: uuid.UUID,
+        datapoints: list[EvaluationResultDatapoint | PartialEvaluationDatapoint],
+        group_name: str | None = None,
+        initial_length: int = INITIAL_EVALUATION_DATAPOINT_MAX_DATA_LENGTH,
+        max_retries: int = 20,
+    ):
+        retry = 0
+        length = initial_length
+        while retry < max_retries:
+            retry += 1
+            length = length // 2
+            logger.debug(
+                f"Retrying save datapoints: {retry} of {max_retries}, length: {length}"
+            )
+            if length == 0:
+                raise ValueError("Error saving evaluation datapoints")
+            points = [
+                datapoint.to_dict(max_data_length=length) for datapoint in datapoints
+            ]
+            response = self._client.post(
+                self._base_url + f"/v1/evals/{eval_id}/datapoints",
+                json={
+                    "points": points,
+                    "groupName": group_name,
+                },
+                headers=self._headers(),
+            )
+            if response.status_code != 413:
+                break
+        if response.status_code != 200:
+            raise ValueError(
+                f"Error saving evaluation datapoints: [{response.status_code}] {response.text}"
+            )

lmnr/sdk/client/synchronous/resources/evaluators.py

@@ -0,0 +1,85 @@
+"""Evaluators resource for creating evaluator scores."""
+
+import uuid
+from typing import Any
+
+from lmnr.sdk.client.synchronous.resources.base import BaseResource
+from lmnr.sdk.utils import format_id
+
+
+class Evaluators(BaseResource):
+    """Resource for creating evaluator scores."""
+
+    def score(
+        self,
+        *,
+        name: str,
+        trace_id: str | int | uuid.UUID | None = None,
+        span_id: str | int | uuid.UUID | None = None,
+        metadata: dict[str, Any] | None = None,
+        score: float,
+    ) -> None:
+        """Create a score for a span.
+
+        Args:
+            name (str): Name of the score
+            trace_id (str | int | uuid.UUID | None, optional): The trace ID to score (will be attached to root span)
+            span_id (str | int | uuid.UUID | None, optional): The span ID to score
+            metadata (dict[str, Any] | None, optional): Additional metadata. Defaults to None.
+            score (float): The score value (float)
+
+        Raises:
+            ValueError: If there's an error creating the score.
+
+        Example:
+            Score by trace ID (will attach to root span):
+
+            >>> laminar_client.evaluators.score(
+            ...     name="quality",
+            ...     trace_id="trace-id-here",
+            ...     score=0.95,
+            ...     metadata={"model": "gpt-4"}
+            ... )
+
+            Score by span ID:
+
+            >>> laminar_client.evaluators.score(
+            ...     name="relevance",
+            ...     span_id="span-id-here", 
+            ...     score=0.87
+            ... )
+        """
+        if trace_id is not None and span_id is not None:
+            raise ValueError("Cannot provide both trace_id and span_id. Please provide only one.")
+        if trace_id is None and span_id is None:
+            raise ValueError("Either 'trace_id' or 'span_id' must be provided.")
+
+        if trace_id is not None:
+            formatted_trace_id = format_id(trace_id)
+            payload = {
+                "name": name,
+                "traceId": formatted_trace_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+        else:
+            formatted_span_id = format_id(span_id)
+            payload = {
+                "name": name,
+                "spanId": formatted_span_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+
+        response = self._client.post(
+            self._base_url + "/v1/evaluators/score",
+            json=payload,
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error creating evaluator score: {response.text}")

lmnr/sdk/client/synchronous/resources/tags.py

@@ -0,0 +1,83 @@
+"""Resource for tagging traces."""
+
+import json
+import uuid
+
+from lmnr.sdk.client.synchronous.resources.base import BaseResource
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.utils import format_id
+
+logger = get_default_logger(__name__)
+
+
+class Tags(BaseResource):
+    """Resource for tagging traces."""
+
+    def tag(
+        self,
+        trace_id: str | int | uuid.UUID,
+        tags: list[str] | str,
+    ):
+        """Tag a trace with a list of tags. Note that the trace must be ended
+        before tagging it. You may want to call `Laminar.flush()` after the
+        trace that you want to tag.
+
+        Args:
+            trace_id (str | int | uuid.UUID): The trace id to tag.
+            tags (list[str] | str): The tag or list of tags to add to the trace.
+
+        Raises:
+            ValueError: If the trace id is not a valid UUID.
+
+        Returns:
+            list[dict]: The response from the server.
+
+        Example:
+        ```python
+        from lmnr import Laminar, LaminarClient, observe
+
+        Laminar.initialize()
+        client = LaminarClient()
+        trace_id = None
+
+        @observe()
+        def foo():
+            trace_id = Laminar.get_trace_id()
+            pass
+
+        # make sure `foo` is called outside a trace context
+        foo()
+
+        # or make sure the trace is ended by this point
+        Laminar.flush()
+
+        client.tags.tag(trace_id, "my_tag")
+        ```
+        """
+        trace_tags = tags if isinstance(tags, list) else [tags]
+        formatted_trace_id = format_id(trace_id)
+
+        url = self._base_url + "/v1/tag"
+        payload = {
+            "traceId": formatted_trace_id,
+            "names": trace_tags,
+        }
+        response = self._client.post(
+            url,
+            content=json.dumps(payload),
+            headers={
+                **self._headers(),
+            },
+        )
+
+        if response.status_code == 404:
+            logger.warning(
+                f"Trace {formatted_trace_id} not found. The trace may have not been ended yet."
+            )
+            return []
+
+        if response.status_code != 200:
+            raise ValueError(
+                f"Failed to tag trace: [{response.status_code}] {response.text}"
+            )
+        return response.json()

lmnr/sdk/client/synchronous/sync_client.py

@@ -0,0 +1,191 @@
+"""
+Laminar HTTP client. Used to send data to/from the Laminar API.
+"""
+
+import httpx
+import re
+from typing import TypeVar
+from types import TracebackType
+
+from lmnr.sdk.client.synchronous.resources import (
+    BrowserEvents,
+    Evals,
+    Evaluators,
+    Tags,
+)
+from lmnr.sdk.client.synchronous.resources.datasets import Datasets
+from lmnr.sdk.utils import from_env
+
+_T = TypeVar("_T", bound="LaminarClient")
+
+
+class LaminarClient:
+    __base_url: str
+    __project_api_key: str
+    __client: httpx.Client = None
+
+    __evals: Evals | None = None
+    __tags: Tags | None = None
+    __evaluators: Evaluators | None = None
+
+    def __init__(
+        self,
+        base_url: str | None = None,
+        project_api_key: str | None = None,
+        port: int | None = None,
+        timeout: int = 3600,
+    ):
+        """Initializer for the Laminar HTTP client.
+
+        Args:
+            base_url (str | None): base URL of the Laminar API. If not\
+                provided, the LMNR_BASE_URL environment variable is used or we\
+                default to "https://api.lmnr.ai".
+            project_api_key (str | None): Laminar project API key. If not\
+                provided, the LMNR_PROJECT_API_KEY environment variable is used.
+            port (int | None, optional): port of the Laminar API HTTP server.\
+                Overrides any port in the base URL.
+                Defaults to None. If none is provided, the default port (443) will
+                be used.
+            timeout (int, optional): global timeout seconds for the HTTP client.\
+                Applied to all httpx operations, i.e. connect, read, get_from_pool, etc.
+                Defaults to 3600.
+        """
+        # If port is already in the base URL, use it as is
+        base_url = base_url or from_env("LMNR_BASE_URL") or "https://api.lmnr.ai"
+        if match := re.search(r":(\d{1,5})$", base_url):
+            base_url = base_url[: -len(match.group(0))]
+            if port is None:
+                port = int(match.group(1))
+
+        base_url = base_url.rstrip("/")
+        self.__base_url = f"{base_url}:{port or 443}"
+        self.__project_api_key = project_api_key or from_env("LMNR_PROJECT_API_KEY")
+        if not self.__project_api_key:
+            raise ValueError(
+                "Project API key is not set. Please set the LMNR_PROJECT_API_KEY environment "
+                "variable or pass project_api_key to the initializer."
+            )
+        self.__client = httpx.Client(
+            headers=self._headers(),
+            timeout=timeout,
+            # Context: If the server responds with a 413, the connection becomes
+            # poisoned and freezes on subsequent requests, and there is no way
+            # to recover or recycle such connection.
+            # Setting max_keepalive_connections to 0 will resolve this, but is
+            # less efficient, as it will create a new connection
+            # (not client, so still better) for each request.
+            #
+            # Note: from my experiments with a simple python server, forcing the
+            # server to read/consume the request payload from the socket seems
+            # to resolve this, but I haven't figured out how to do that in our
+            # real actix-web backend server and whether it makes sense to do so.
+            #
+            # TODO: investigate if there are better ways to fix this rather than
+            # setting keepalive_expiry to 0. Other alternative: migrate to
+            # requests + aiohttp.
+            #
+            # limits=httpx.Limits(
+            #     max_keepalive_connections=0,
+            #     keepalive_expiry=0,
+            # ),
+        )
+
+        # Initialize resource objects
+        self.__evals = Evals(self.__client, self.__base_url, self.__project_api_key)
+        self.__evaluators = Evaluators(
+            self.__client, self.__base_url, self.__project_api_key
+        )
+        self.__browser_events = BrowserEvents(
+            self.__client, self.__base_url, self.__project_api_key
+        )
+        self.__tags = Tags(self.__client, self.__base_url, self.__project_api_key)
+        self.__datasets = Datasets(
+            self.__client, self.__base_url, self.__project_api_key
+        )
+
+    @property
+    def evals(self) -> Evals:
+        """Get the Evals resource.
+
+        Returns:
+            Evals: The Evals resource instance.
+        """
+        return self.__evals
+
+    @property
+    def _browser_events(self) -> BrowserEvents:
+        """Get the BrowserEvents resource.
+
+        Returns:
+            BrowserEvents: The BrowserEvents resource instance.
+        """
+        return self.__browser_events
+
+    @property
+    def tags(self) -> Tags:
+        """Get the Tags resource.
+
+        Returns:
+            Tags: The Tags resource instance.
+        """
+        return self.__tags
+
+    @property
+    def evaluators(self) -> Evaluators:
+        """Get the Evaluators resource.
+
+        Returns:
+            Evaluators: The Evaluators resource instance.
+        """
+        return self.__evaluators
+
+    @property
+    def datasets(self) -> Datasets:
+        """Get the Datasets resource.
+
+        Returns:
+            Datasets: The Datasets resource instance.
+        """
+        return self.__datasets
+
+    def shutdown(self):
+        """Shutdown the client by closing underlying connections."""
+        self.__client.close()
+
+    def is_closed(self) -> bool:
+        """Check if the client is closed.
+
+        Returns:
+            bool: True if the client is closed, False otherwise.
+        """
+        return self.__client.is_closed
+
+    def close(self) -> None:
+        """Close the underlying HTTPX client.
+
+        The client will *not* be usable after this.
+        """
+        # If an error is thrown while constructing a client, self._client
+        # may not be present
+        if hasattr(self, "__client"):
+            self.__client.close()
+
+    def __enter__(self: _T) -> _T:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        exc_tb: TracebackType | None,
+    ) -> None:
+        self.close()
+
+    def _headers(self) -> dict[str, str]:
+        assert self.__project_api_key is not None, "Project API key is not set"
+        return {
+            "Authorization": "Bearer " + self.__project_api_key,
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+        }