lmnr 0.4.66__py3-none-any.whl → 0.5.1__py3-none-any.whl

lmnr/sdk/client/synchronous/sync_client.py

@@ -0,0 +1,170 @@
+"""
+Laminar HTTP client. Used to send data to/from the Laminar API.
+"""
+
+import httpx
+import re
+from typing import Optional, TypeVar
+from types import TracebackType
+
+from lmnr.sdk.client.synchronous.resources import (
+    Agent,
+    BrowserEvents,
+    Evals,
+    Pipeline,
+    SemanticSearch,
+)
+from lmnr.sdk.utils import from_env
+
+_T = TypeVar("_T", bound="LaminarClient")
+
+
+class LaminarClient:
+    __base_url: str
+    __project_api_key: str
+    __client: httpx.Client = None
+
+    # Resource properties
+    __pipeline: Optional[Pipeline] = None
+    __semantic_search: Optional[SemanticSearch] = None
+    __agent: Optional[Agent] = None
+    __evals: Optional[Evals] = None
+
+    def __init__(
+        self,
+        base_url: Optional[str] = None,
+        project_api_key: Optional[str] = None,
+        port: Optional[int] = None,
+        timeout: int = 3600,
+    ):
+        """Initializer for the Laminar HTTP client.
+
+        Args:
+            base_url (str): base URL of the Laminar API.
+            project_api_key (str): Laminar project API key
+            port (Optional[int], optional): port of the Laminar API HTTP server.\
+                Overrides any port in the base URL.
+                Defaults to None. If none is provided, the default port (443) will
+                be used.
+            timeout (int, optional): global timeout seconds for the HTTP client.\
+                Applied to all httpx operations, i.e. connect, read, get_from_pool, etc.
+                Defaults to 3600.
+        """
+        # If port is already in the base URL, use it as is
+        base_url = base_url or from_env("LMNR_BASE_URL") or "https://api.lmnr.ai"
+        if match := re.search(r":(\d{1,5})$", base_url):
+            base_url = base_url[: -len(match.group(0))]
+            if port is None:
+                port = int(match.group(1))
+
+        base_url = base_url.rstrip("/")
+        self.__base_url = f"{base_url}:{port or 443}"
+        self.__project_api_key = project_api_key or from_env("LMNR_PROJECT_API_KEY")
+        if not self.__project_api_key:
+            raise ValueError(
+                "Project API key is not set. Please set the LMNR_PROJECT_API_KEY environment "
+                "variable or pass project_api_key to the initializer."
+            )
+        self.__client = httpx.Client(
+            headers=self._headers(),
+            timeout=timeout,
+        )
+
+        # Initialize resource objects
+        self.__pipeline = Pipeline(
+            self.__client, self.__base_url, self.__project_api_key
+        )
+        self.__semantic_search = SemanticSearch(
+            self.__client, self.__base_url, self.__project_api_key
+        )
+        self.__agent = Agent(self.__client, self.__base_url, self.__project_api_key)
+        self.__evals = Evals(self.__client, self.__base_url, self.__project_api_key)
+        self.__browser_events = BrowserEvents(
+            self.__client, self.__base_url, self.__project_api_key
+        )
+
+    @property
+    def pipeline(self) -> Pipeline:
+        """Get the Pipeline resource.
+
+        Returns:
+            Pipeline: The Pipeline resource instance.
+        """
+        return self.__pipeline
+
+    @property
+    def semantic_search(self) -> SemanticSearch:
+        """Get the SemanticSearch resource.
+
+        Returns:
+            SemanticSearch: The SemanticSearch resource instance.
+        """
+        return self.__semantic_search
+
+    @property
+    def agent(self) -> Agent:
+        """Get the Agent resource.
+
+        Returns:
+            Agent: The Agent resource instance.
+        """
+        return self.__agent
+
+    @property
+    def _evals(self) -> Evals:
+        """Get the Evals resource.
+
+        Returns:
+            Evals: The Evals resource instance.
+        """
+        return self.__evals
+
+    @property
+    def _browser_events(self) -> BrowserEvents:
+        """Get the BrowserEvents resource.
+
+        Returns:
+            BrowserEvents: The BrowserEvents resource instance.
+        """
+        return self.__browser_events
+
+    def shutdown(self):
+        """Shutdown the client by closing underlying connections."""
+        self.__client.close()
+
+    def is_closed(self) -> bool:
+        """Check if the client is closed.
+
+        Returns:
+            bool: True if the client is closed, False otherwise.
+        """
+        return self.__client.is_closed
+
+    def close(self) -> None:
+        """Close the underlying HTTPX client.
+
+        The client will *not* be usable after this.
+        """
+        # If an error is thrown while constructing a client, self._client
+        # may not be present
+        if hasattr(self, "_client"):
+            self.__client.close()
+
+    def __enter__(self: _T) -> _T:
+        return self
+
+    def __exit__(
+        self,
+        exc_type: Optional[type[BaseException]],
+        exc: Optional[BaseException],
+        exc_tb: Optional[TracebackType],
+    ) -> None:
+        self.close()
+
+    def _headers(self) -> dict[str, str]:
+        assert self.__project_api_key is not None, "Project API key is not set"
+        return {
+            "Authorization": "Bearer " + self.__project_api_key,
+            "Content-Type": "application/json",
+            "Accept": "application/json",
+        }

lmnr/sdk/datasets.py

@@ -1,6 +1,6 @@
 from abc import ABC, abstractmethod
 
-from .client import LaminarClient
+from .client.synchronous.sync_client import LaminarClient
 from .log import get_default_logger
 from .types import Datapoint
 
@@ -38,7 +38,9 @@ class LaminarDataset(EvaluationDataset):
             f"dataset {self.name}. Fetching batch from {self._offset} to "
             + f"{self._offset + self._fetch_size}"
         )
-        resp = LaminarClient.get_datapoints(self.name, self._offset, self._fetch_size)
+        resp = self.client._evals.get_datapoints(
+            self.name, self._offset, self._fetch_size
+        )
         self._fetched_items += resp.items
         self._offset = len(self._fetched_items)
         if self._len is None:
@@ -53,3 +55,6 @@ class LaminarDataset(EvaluationDataset):
         if idx >= len(self._fetched_items):
             self._fetch_batch()
         return self._fetched_items[idx]
+
+    def set_client(self, client: LaminarClient):
+        self.client = client

lmnr/sdk/evaluations.py

@@ -1,19 +1,20 @@
 import asyncio
 import re
-import sys
 import uuid
+import dotenv
 from tqdm import tqdm
 from typing import Any, Awaitable, Optional, Set, Union
 
-from ..openllmetry_sdk.instruments import Instruments
-from ..openllmetry_sdk.tracing.attributes import SPAN_TYPE
+from lmnr.openllmetry_sdk.instruments import Instruments
+from lmnr.openllmetry_sdk.tracing.attributes import SPAN_TYPE
 
-from .client import LaminarClient
-from .datasets import EvaluationDataset
-from .eval_control import EVALUATION_INSTANCE, PREPARE_ONLY
-from .laminar import Laminar as L
-from .log import get_default_logger
-from .types import (
+from lmnr.sdk.client.asynchronous.async_client import AsyncLaminarClient
+from lmnr.sdk.client.synchronous.sync_client import LaminarClient
+from lmnr.sdk.datasets import EvaluationDataset, LaminarDataset
+from lmnr.sdk.eval_control import EVALUATION_INSTANCE, PREPARE_ONLY
+from lmnr.sdk.laminar import Laminar as L
+from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.types import (
     Datapoint,
     EvaluationResultDatapoint,
     EvaluatorFunction,
@@ -25,7 +26,7 @@ from .types import (
     SpanType,
     TraceType,
 )
-from .utils import is_async
+from lmnr.sdk.utils import from_env, is_async
 
 DEFAULT_BATCH_SIZE = 5
 MAX_EXPORT_BATCH_SIZE = 64
@@ -34,15 +35,13 @@ MAX_EXPORT_BATCH_SIZE = 64
 def get_evaluation_url(
     project_id: str, evaluation_id: str, base_url: Optional[str] = None
 ):
-    if not base_url:
+    if not base_url or base_url == "https://api.lmnr.ai":
         base_url = "https://www.lmnr.ai"
 
     url = base_url
-    if url.endswith("/"):
-        url = url[:-1]
+    url = re.sub(r"\/$", "", url)
     if url.endswith("localhost") or url.endswith("127.0.0.1"):
-        # We best effort assume that the frontend is running on port 3000
-        # TODO: expose the frontend port?
+        # We best effort assume that the frontend is running on port 5667
         url = url + ":5667"
     return f"{url}/project/{project_id}/evaluations/{evaluation_id}"
 
@@ -78,7 +77,7 @@ class EvaluationReporter:
 
     def stopWithError(self, error: Exception):
         self.cli_progress.close()
-        sys.stderr.write(f"\nError: {error}\n")
+        raise error
 
     def stop(
         self, average_scores: dict[str, Numeric], project_id: str, evaluation_id: str
@@ -175,6 +174,8 @@ class Evaluation:
                     "underscores, or spaces."
                 )
 
+        base_url = base_url or from_env("LMNR_BASE_URL") or "https://api.lmnr.ai"
+
         self.is_finished = False
         self.reporter = EvaluationReporter(base_url)
         if isinstance(data, list):
@@ -192,7 +193,27 @@ class Evaluation:
         self.batch_size = concurrency_limit
         self._logger = get_default_logger(self.__class__.__name__)
         self.human_evaluators = human_evaluators
-        self.upload_tasks = []  # Add this line to track upload tasks
+        self.upload_tasks = []
+        self.base_http_url = f"{base_url}:{http_port or 443}"
+
+        api_key = project_api_key
+        if not api_key:
+            dotenv_path = dotenv.find_dotenv(usecwd=True)
+            api_key = dotenv.get_key(
+                dotenv_path=dotenv_path, key_to_get="LMNR_PROJECT_API_KEY"
+            )
+        if not api_key:
+            raise ValueError(
+                "Please initialize the Laminar object with"
+                " your project API key or set the LMNR_PROJECT_API_KEY"
+                " environment variable in your environment or .env file"
+            )
+        self.project_api_key = api_key
+
+        self.client = AsyncLaminarClient(
+            base_url=self.base_http_url,
+            project_api_key=self.project_api_key,
+        )
         L.initialize(
             project_api_key=project_api_key,
             base_url=base_url,
@@ -209,9 +230,16 @@ class Evaluation:
         return await self._run()
 
     async def _run(self) -> None:
+        if isinstance(self.data, LaminarDataset):
+            self.data.set_client(
+                LaminarClient(
+                    self.base_http_url,
+                    self.project_api_key,
+                )
+            )
         self.reporter.start(len(self.data))
         try:
-            evaluation = await LaminarClient.init_eval(
+            evaluation = await self.client._evals.init(
                 name=self.name, group_name=self.group_name
             )
             result_datapoints = await self._evaluate_in_batches(evaluation.id)
@@ -226,12 +254,19 @@ class Evaluation:
         except Exception as e:
             self.reporter.stopWithError(e)
             self.is_finished = True
+            await self._shutdown()
             return
 
         average_scores = get_average_scores(result_datapoints)
         self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
         self.is_finished = True
-        await LaminarClient.shutdown_async()
+        await self._shutdown()
+
+    async def _shutdown(self):
+        L.shutdown()
+        await self.client.close()
+        if isinstance(self.data, LaminarDataset) and self.data.client:
+            self.data.client.close()
 
     async def _evaluate_in_batches(
         self, eval_id: uuid.UUID
@@ -285,7 +320,7 @@ class Evaluation:
                     executor_span_id=executor_span_id,
                 )
                 # First, create datapoint with trace_id so that we can show the dp in the UI
-                await LaminarClient.save_eval_datapoints(
+                await self.client._evals.save_datapoints(
                     eval_id, [partial_datapoint], self.group_name
                 )
                 executor_span.set_attribute(SPAN_TYPE, SpanType.EXECUTOR.value)
@@ -342,7 +377,7 @@ class Evaluation:
 
         # Create background upload task without awaiting it
         upload_task = asyncio.create_task(
-            LaminarClient.save_eval_datapoints(eval_id, [datapoint], self.group_name)
+            self.client._evals.save_datapoints(eval_id, [datapoint], self.group_name)
         )
         self.upload_tasks.append(upload_task)
 
@@ -355,7 +390,6 @@ def evaluate(
     evaluators: dict[str, EvaluatorFunction],
     human_evaluators: list[HumanEvaluator] = [],
     name: Optional[str] = None,
-    group_id: Optional[str] = None,  # Deprecated
     group_name: Optional[str] = None,
     concurrency_limit: int = DEFAULT_BATCH_SIZE,
     project_api_key: Optional[str] = None,
@@ -372,8 +406,8 @@ def evaluate(
 
     If there is no event loop, creates it and runs the evaluation until
     completion.
-    If there is an event loop, schedules the evaluation as a task in the
-    event loop and returns an awaitable handle.
+    If there is an event loop, returns an awaitable handle immediately. IMPORTANT:
+    You must await the call to `evaluate`.
 
     Parameters:
         data (Union[list[EvaluationDatapoint|dict]], EvaluationDataset]):\
@@ -399,11 +433,6 @@ def evaluate(
             Used to identify the evaluation in the group. If not provided, a\
             random name will be generated.
             Defaults to None.
-        group_id (Optional[str], optional): [DEPRECATED] Use group_name instead.
-                        An identifier to group evaluations.\
-                        Only evaluations within the same group_id can be\
-                        visually compared. If not provided, set to "default".
-                        Defaults to None
         group_name (Optional[str], optional): An identifier to group evaluations.\
             Only evaluations within the same group_name can be visually compared.\
             If not provided, set to "default".
@@ -429,11 +458,6 @@ def evaluate(
         trace_export_timeout_seconds (Optional[int], optional): The timeout for\
                         trace export on OpenTelemetry exporter. Defaults to None.
     """
-    if group_id:
-        raise DeprecationWarning("group_id is deprecated. Use group_name instead.")
-
-    group_name = group_name or group_id
-
     evaluation = Evaluation(
         data=data,
         executor=executor,
@@ -456,6 +480,6 @@ def evaluate(
     else:
         loop = asyncio.get_event_loop()
         if loop.is_running():
-            return loop.run_until_complete(evaluation.run())
+            return evaluation.run()
         else:
             return asyncio.run(evaluation.run())