lmnr 0.5.0__py3-none-any.whl → 0.5.1a0__py3-none-any.whl

lmnr/sdk/browser/playwright_otel.py

@@ -2,9 +2,7 @@ import logging
 import uuid
 
 from lmnr.sdk.browser.pw_utils import handle_navigation_async, handle_navigation_sync
-from lmnr.sdk.browser.utils import with_tracer_and_client_wrapper
-from lmnr.sdk.client.asynchronous.async_client import AsyncLaminarClient
-from lmnr.sdk.client.synchronous.sync_client import LaminarClient
+from lmnr.sdk.browser.utils import with_tracer_wrapper
 from lmnr.version import __version__
 
 from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
@@ -18,12 +16,15 @@ from opentelemetry.trace import (
     set_span_in_context,
 )
 from opentelemetry.context import get_current
-from typing import Collection
+from typing import Collection, Optional
 from wrapt import wrap_function_wrapper
 
 try:
     from playwright.async_api import Browser
-    from playwright.sync_api import Browser as SyncBrowser
+    from playwright.sync_api import (
+        Browser as SyncBrowser,
+        BrowserContext as SyncBrowserContext,
+    )
 except ImportError as e:
     raise ImportError(
         f"Attempted to import {__file__}, but it is designed "
@@ -36,12 +37,12 @@ _instruments = ("playwright >= 1.9.0",)
 logger = logging.getLogger(__name__)
 
 _context_spans: dict[str, Span] = {}
+_project_api_key: Optional[str] = None
+_base_http_url: Optional[str] = None
 
 
-@with_tracer_and_client_wrapper
-def _wrap_new_page(
-    tracer: Tracer, client: LaminarClient, to_wrap, wrapped, instance, args, kwargs
-):
+@with_tracer_wrapper
+def _wrap_new_page(tracer: Tracer, to_wrap, wrapped, instance, args, kwargs):
     with tracer.start_as_current_span(
         f"{to_wrap.get('object')}.{to_wrap.get('method')}"
     ) as span:
@@ -49,13 +50,15 @@ def _wrap_new_page(
         session_id = str(uuid.uuid4().hex)
         trace_id = format(get_current_span().get_span_context().trace_id, "032x")
         span.set_attribute("lmnr.internal.has_browser_session", True)
-        handle_navigation_sync(page, session_id, trace_id, client)
+        handle_navigation_sync(
+            page, session_id, trace_id, _project_api_key, _base_http_url
+        )
         return page
 
 
-@with_tracer_and_client_wrapper
+@with_tracer_wrapper
 async def _wrap_new_page_async(
-    tracer: Tracer, client: AsyncLaminarClient, to_wrap, wrapped, instance, args, kwargs
+    tracer: Tracer, to_wrap, wrapped, instance, args, kwargs
 ):
     with tracer.start_as_current_span(
         f"{to_wrap.get('object')}.{to_wrap.get('method')}"
@@ -64,14 +67,14 @@ async def _wrap_new_page_async(
         session_id = str(uuid.uuid4().hex)
         trace_id = format(span.get_span_context().trace_id, "032x")
         span.set_attribute("lmnr.internal.has_browser_session", True)
-        await handle_navigation_async(page, session_id, trace_id, client)
+        await handle_navigation_async(
+            page, session_id, trace_id, _project_api_key, _base_http_url
+        )
         return page
 
 
-@with_tracer_and_client_wrapper
-def _wrap_new_browser_sync(
-    tracer: Tracer, client: LaminarClient, to_wrap, wrapped, instance, args, kwargs
-):
+@with_tracer_wrapper
+def _wrap_new_browser_sync(tracer: Tracer, to_wrap, wrapped, instance, args, kwargs):
     global _context_spans
     browser: SyncBrowser = wrapped(*args, **kwargs)
     session_id = str(uuid.uuid4().hex)
@@ -84,15 +87,23 @@ def _wrap_new_browser_sync(
             set_span_in_context(span, get_current())
             _context_spans[id(context)] = span
         span.set_attribute("lmnr.internal.has_browser_session", True)
+        trace_id = format(span.get_span_context().trace_id, "032x")
+        context.on(
+            "page",
+            lambda page: handle_navigation_sync(
+                page, session_id, trace_id, _project_api_key, _base_http_url
+            ),
+        )
         for page in context.pages:
-            trace_id = format(span.get_span_context().trace_id, "032x")
-            handle_navigation_sync(page, session_id, trace_id, client)
+            handle_navigation_sync(
+                page, session_id, trace_id, _project_api_key, _base_http_url
+            )
     return browser
 
 
-@with_tracer_and_client_wrapper
+@with_tracer_wrapper
 async def _wrap_new_browser_async(
-    tracer: Tracer, client: AsyncLaminarClient, to_wrap, wrapped, instance, args, kwargs
+    tracer: Tracer, to_wrap, wrapped, instance, args, kwargs
 ):
     global _context_spans
     browser: Browser = await wrapped(*args, **kwargs)
@@ -106,16 +117,80 @@ async def _wrap_new_browser_async(
             set_span_in_context(span, get_current())
             _context_spans[id(context)] = span
         span.set_attribute("lmnr.internal.has_browser_session", True)
+        trace_id = format(span.get_span_context().trace_id, "032x")
+
+        async def handle_page_navigation(page):
+            return await handle_navigation_async(
+                page, session_id, trace_id, _project_api_key, _base_http_url
+            )
+
+        context.on("page", handle_page_navigation)
         for page in context.pages:
-            trace_id = format(span.get_span_context().trace_id, "032x")
-            await handle_navigation_async(page, session_id, trace_id, client)
+            await handle_navigation_async(
+                page, session_id, trace_id, _project_api_key, _base_http_url
+            )
     return browser
 
 
-@with_tracer_and_client_wrapper
+@with_tracer_wrapper
+def _wrap_new_context_sync(tracer: Tracer, to_wrap, wrapped, instance, args, kwargs):
+    context: SyncBrowserContext = wrapped(*args, **kwargs)
+    session_id = str(uuid.uuid4().hex)
+    span = get_current_span()
+    if span == INVALID_SPAN:
+        span = tracer.start_span(
+            name=f"{to_wrap.get('object')}.{to_wrap.get('method')}"
+        )
+        set_span_in_context(span, get_current())
+        _context_spans[id(context)] = span
+    span.set_attribute("lmnr.internal.has_browser_session", True)
+    trace_id = format(span.get_span_context().trace_id, "032x")
+
+    context.on(
+        "page",
+        lambda page: handle_navigation_sync(
+            page, session_id, trace_id, _project_api_key, _base_http_url
+        ),
+    )
+    for page in context.pages:
+        handle_navigation_sync(
+            page, session_id, trace_id, _project_api_key, _base_http_url
+        )
+    return context
+
+
+@with_tracer_wrapper
+async def _wrap_new_context_async(
+    tracer: Tracer, to_wrap, wrapped, instance, args, kwargs
+):
+    context: SyncBrowserContext = await wrapped(*args, **kwargs)
+    session_id = str(uuid.uuid4().hex)
+    span = get_current_span()
+    if span == INVALID_SPAN:
+        span = tracer.start_span(
+            name=f"{to_wrap.get('object')}.{to_wrap.get('method')}"
+        )
+        set_span_in_context(span, get_current())
+        _context_spans[id(context)] = span
+    span.set_attribute("lmnr.internal.has_browser_session", True)
+    trace_id = format(span.get_span_context().trace_id, "032x")
+
+    async def handle_page_navigation(page):
+        return await handle_navigation_async(
+            page, session_id, trace_id, _project_api_key, _base_http_url
+        )
+
+    context.on("page", handle_page_navigation)
+    for page in context.pages:
+        await handle_navigation_async(
+            page, session_id, trace_id, _project_api_key, _base_http_url
+        )
+    return context
+
+
+@with_tracer_wrapper
 def _wrap_close_browser_sync(
     tracer: Tracer,
-    client: LaminarClient,
     to_wrap,
     wrapped,
     instance: SyncBrowser,
@@ -133,10 +208,9 @@ def _wrap_close_browser_sync(
     return wrapped(*args, **kwargs)
 
 
-@with_tracer_and_client_wrapper
+@with_tracer_wrapper
 async def _wrap_close_browser_async(
     tracer: Tracer,
-    client: AsyncLaminarClient,
     to_wrap,
     wrapped,
     instance: Browser,
@@ -191,6 +265,18 @@ WRAPPED_METHODS = [
         "method": "close",
         "wrapper": _wrap_close_browser_sync,
     },
+    {
+        "package": "playwright.sync_api",
+        "object": "Browser",
+        "method": "new_context",
+        "wrapper": _wrap_new_context_sync,
+    },
+    {
+        "package": "playwright.sync_api",
+        "object": "BrowserType",
+        "method": "launch_persistent_context",
+        "wrapper": _wrap_new_context_sync,
+    },
 ]
 
 WRAPPED_METHODS_ASYNC = [
@@ -230,22 +316,40 @@ WRAPPED_METHODS_ASYNC = [
         "method": "close",
         "wrapper": _wrap_close_browser_async,
     },
+    {
+        "package": "playwright.async_api",
+        "object": "Browser",
+        "method": "new_context",
+        "wrapper": _wrap_new_context_async,
+    },
+    {
+        "package": "playwright.sync_api",
+        "object": "BrowserType",
+        "method": "launch_persistent_context",
+        "wrapper": _wrap_new_context_sync,
+    },
 ]
 
 
 class PlaywrightInstrumentor(BaseInstrumentor):
-    def __init__(self, client: LaminarClient, async_client: AsyncLaminarClient):
+    def __init__(self):
         super().__init__()
-        self.client = client
-        self.async_client = async_client
 
     def instrumentation_dependencies(self) -> Collection[str]:
         return _instruments
 
     def _instrument(self, **kwargs):
+        global _project_api_key, _base_http_url
+
         tracer_provider = kwargs.get("tracer_provider")
         tracer = get_tracer(__name__, __version__, tracer_provider)
 
+        if kwargs.get("project_api_key"):
+            _project_api_key = kwargs.get("project_api_key")
+
+        if kwargs.get("base_http_url"):
+            _base_http_url = kwargs.get("base_http_url")
+
         for wrapped_method in WRAPPED_METHODS:
             wrap_package = wrapped_method.get("package")
             wrap_object = wrapped_method.get("object")
@@ -256,7 +360,6 @@ class PlaywrightInstrumentor(BaseInstrumentor):
                     f"{wrap_object}.{wrap_method}",
                     wrapped_method.get("wrapper")(
                         tracer,
-                        self.client,
                         wrapped_method,
                     ),
                 )
@@ -274,7 +377,6 @@ class PlaywrightInstrumentor(BaseInstrumentor):
                     f"{wrap_object}.{wrap_method}",
                     wrapped_method.get("wrapper")(
                         tracer,
-                        self.async_client,
                         wrapped_method,
                     ),
                 )

lmnr/sdk/browser/pw_utils.py

@@ -3,6 +3,7 @@ import logging
 import os
 import time
 import threading
+from typing import Optional
 
 from opentelemetry import trace
 
@@ -26,6 +27,8 @@ logger = logging.getLogger(__name__)
 # Track pages we've already instrumented to avoid double-instrumentation
 instrumented_pages = set()
 async_instrumented_pages = set()
+client: Optional[LaminarClient] = None
+async_client: Optional[AsyncLaminarClient] = None
 
 
 current_dir = os.path.dirname(os.path.abspath(__file__))
@@ -36,7 +39,7 @@ INJECT_PLACEHOLDER = """
 () => {
     const BATCH_SIZE = 1000;  // Maximum events to store in memory
     
-    window.lmnrRrwebEventsBatch = [];
+    window.lmnrRrwebEventsBatch = new Set();
     
     // Utility function to compress individual event data
     async function compressEventData(data) {
@@ -50,8 +53,8 @@ INJECT_PLACEHOLDER = """
     
     window.lmnrGetAndClearEvents = () => {
         const events = window.lmnrRrwebEventsBatch;
-        window.lmnrRrwebEventsBatch = [];
-        return events;
+        window.lmnrRrwebEventsBatch = new Set();
+        return Array.from(events);
     };
 
     // Add heartbeat events
@@ -62,11 +65,11 @@ INJECT_PLACEHOLDER = """
             timestamp: Date.now()
         };
         
-        window.lmnrRrwebEventsBatch.push(heartbeat);
+        window.lmnrRrwebEventsBatch.add(heartbeat);
         
         // Prevent memory issues by limiting batch size
-        if (window.lmnrRrwebEventsBatch.length > BATCH_SIZE) {
-            window.lmnrRrwebEventsBatch = window.lmnrRrwebEventsBatch.slice(-BATCH_SIZE);
+        if (window.lmnrRrwebEventsBatch.size > BATCH_SIZE) {
+            window.lmnrRrwebEventsBatch = new Set(Array.from(window.lmnrRrwebEventsBatch).slice(-BATCH_SIZE));
         }
     }, 1000);
 
@@ -81,7 +84,7 @@ INJECT_PLACEHOLDER = """
                 ...event,
                 data: await compressEventData(event.data)
             };
-            window.lmnrRrwebEventsBatch.push(compressedEvent);
+            window.lmnrRrwebEventsBatch.add(compressedEvent);
         }
     });
 }
@@ -220,8 +223,21 @@ async def inject_rrweb_async(page: Page):
 
 @observe(name="playwright.page", ignore_input=True, ignore_output=True)
 def handle_navigation_sync(
-    page: SyncPage, session_id: str, trace_id: str, client: LaminarClient
+    page: SyncPage,
+    session_id: str,
+    trace_id: str,
+    project_api_key: Optional[str] = None,
+    base_http_url: Optional[str] = None,
 ):
+    global client
+    if client is None:
+        client = LaminarClient(base_url=base_http_url, project_api_key=project_api_key)
+    if (base_http_url is not None and base_http_url != client.base_url) or (
+        project_api_key is not None and project_api_key != client.project_api_key
+    ):
+        if client is not None:
+            client.close()
+        client = LaminarClient(base_url=base_http_url, project_api_key=project_api_key)
     trace.get_current_span().set_attribute("lmnr.internal.has_browser_session", True)
     # Check if we've already instrumented this page
     page_id = id(page)
@@ -239,11 +255,10 @@ def handle_navigation_sync(
     inject_rrweb_sync(page)
 
     def collection_loop():
-        while not page.is_closed():  # Stop when page closes
+        while not page.is_closed():
             send_events_sync(page, session_id, trace_id, client)
             time.sleep(2)
 
-        # Clean up when page closes
         if page_id in instrumented_pages:
             instrumented_pages.remove(page_id)
 
@@ -253,8 +268,25 @@ def handle_navigation_sync(
 
 @observe(name="playwright.page", ignore_input=True, ignore_output=True)
 async def handle_navigation_async(
-    page: Page, session_id: str, trace_id: str, client: AsyncLaminarClient
+    page: Page,
+    session_id: str,
+    trace_id: str,
+    project_api_key: Optional[str] = None,
+    base_http_url: Optional[str] = None,
 ):
+    global async_client
+    if async_client is None:
+        async_client = AsyncLaminarClient(
+            base_url=base_http_url, project_api_key=project_api_key
+        )
+    if (base_http_url is not None and base_http_url != async_client.base_url) or (
+        project_api_key is not None and project_api_key != async_client.project_api_key
+    ):
+        if async_client is not None:
+            await async_client.close()
+        async_client = AsyncLaminarClient(
+            base_url=base_http_url, project_api_key=project_api_key
+        )
     trace.get_current_span().set_attribute("lmnr.internal.has_browser_session", True)
     # Check if we've already instrumented this page
     page_id = id(page)
@@ -271,19 +303,19 @@ async def handle_navigation_async(
     page.on("load", lambda: asyncio.create_task(on_load()))
     await inject_rrweb_async(page)
 
-    async def collection_loop():
+    async def collection_loop(client: AsyncLaminarClient):
         try:
-            while not page.is_closed():  # Stop when page closes
+            while not page.is_closed():
                 await send_events_async(page, session_id, trace_id, client)
                 await asyncio.sleep(2)
-            # Clean up when page closes
+
             async_instrumented_pages.remove(page_id)
             logger.info("Event collection stopped")
         except Exception as e:
             logger.error(f"Event collection stopped: {e}")
 
     # Create and store task
-    task = asyncio.create_task(collection_loop())
+    task = asyncio.create_task(collection_loop(async_client))
 
     # Clean up task when page closes
     page.on("close", lambda: task.cancel())

lmnr/sdk/client/asynchronous/async_client.py

@@ -137,6 +137,14 @@ class AsyncLaminarClient:
         """
         await self.__client.aclose()
 
+    @property
+    def base_url(self) -> str:
+        return self.__base_url
+
+    @property
+    def project_api_key(self) -> str:
+        return self.__project_api_key
+
     async def __aenter__(self: _T) -> _T:
         return self
 

lmnr/sdk/client/asynchronous/resources/agent.py

@@ -35,6 +35,7 @@ class AsyncAgent(BaseAsyncResource):
         model_provider: Optional[ModelProvider] = None,
         model: Optional[str] = None,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> AsyncIterator[RunAgentResponseChunk]:
         """Run Laminar index agent in streaming mode.
 
@@ -45,7 +46,7 @@ class AsyncAgent(BaseAsyncResource):
             model_provider (Optional[ModelProvider], optional): LLM model provider
             model (Optional[str], optional): LLM model name
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
-
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
         Returns:
             AsyncIterator[RunAgentResponseChunk]: a generator of response chunks
         """
@@ -59,6 +60,7 @@ class AsyncAgent(BaseAsyncResource):
         model_provider: Optional[ModelProvider] = None,
         model: Optional[str] = None,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> AgentOutput:
         """Run Laminar index agent.
 
@@ -68,7 +70,7 @@ class AsyncAgent(BaseAsyncResource):
             model_provider (Optional[ModelProvider], optional): LLM model provider
             model (Optional[str], optional): LLM model name
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
-
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
         Returns:
             AgentOutput: agent output
         """
@@ -83,6 +85,7 @@ class AsyncAgent(BaseAsyncResource):
         model: Optional[str] = None,
         stream: Literal[False] = False,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> AgentOutput:
         """Run Laminar index agent.
 
@@ -93,7 +96,7 @@ class AsyncAgent(BaseAsyncResource):
             model (Optional[str], optional): LLM model name
             stream (Literal[False], optional): whether to stream the agent's response
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
-
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
         Returns:
             AgentOutput: agent output
         """
@@ -107,6 +110,7 @@ class AsyncAgent(BaseAsyncResource):
         model: Optional[str] = None,
         stream: bool = False,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> Union[AgentOutput, Awaitable[AsyncIterator[RunAgentResponseChunk]]]:
         """Run Laminar index agent.
 
@@ -117,7 +121,7 @@ class AsyncAgent(BaseAsyncResource):
             model (Optional[str], optional): LLM model name
             stream (bool, optional): whether to stream the agent's response
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
-
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
         Returns:
             Union[AgentOutput, AsyncIterator[RunAgentResponseChunk]]: agent output or a generator of response chunks
         """
@@ -146,6 +150,7 @@ class AsyncAgent(BaseAsyncResource):
             # https://aws.amazon.com/blogs/networking-and-content-delivery/introducing-nlb-tcp-configurable-idle-timeout/
             stream=True,
             enable_thinking=enable_thinking,
+            return_screenshots=return_screenshots,
         )
 
         # For streaming case, use a generator function

lmnr/sdk/client/synchronous/resources/agent.py

@@ -27,6 +27,7 @@ class Agent(BaseResource):
         model_provider: Optional[ModelProvider] = None,
         model: Optional[str] = None,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> Generator[RunAgentResponseChunk, None, None]:
         """Run Laminar index agent in streaming mode.
 
@@ -37,7 +38,7 @@ class Agent(BaseResource):
             model_provider (Optional[ModelProvider], optional): LLM model provider
             model (Optional[str], optional): LLM model name
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
-
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
         Returns:
             Generator[RunAgentResponseChunk, None, None]: a generator of response chunks
         """
@@ -51,6 +52,7 @@ class Agent(BaseResource):
         model_provider: Optional[ModelProvider] = None,
         model: Optional[str] = None,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> AgentOutput:
         """Run Laminar index agent.
 
@@ -60,6 +62,7 @@ class Agent(BaseResource):
             model_provider (Optional[ModelProvider], optional): LLM model provider
             model (Optional[str], optional): LLM model name
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
 
         Returns:
             AgentOutput: agent output
@@ -75,6 +78,7 @@ class Agent(BaseResource):
         model: Optional[str] = None,
         stream: Literal[False] = False,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> AgentOutput:
         """Run Laminar index agent.
 
@@ -85,7 +89,7 @@ class Agent(BaseResource):
             model (Optional[str], optional): LLM model name
             stream (Literal[False], optional): whether to stream the agent's response
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
-            cdp_url (Optional[str], optional): CDP URL to connect to an existing browser session.
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
 
         Returns:
             AgentOutput: agent output
@@ -100,6 +104,7 @@ class Agent(BaseResource):
         model: Optional[str] = None,
         stream: bool = False,
         enable_thinking: bool = True,
+        return_screenshots: bool = False,
     ) -> Union[AgentOutput, Generator[RunAgentResponseChunk, None, None]]:
         """Run Laminar index agent.
 
@@ -110,7 +115,7 @@ class Agent(BaseResource):
             model (Optional[str], optional): LLM model name
             stream (bool, optional): whether to stream the agent's response
             enable_thinking (bool, optional): whether to enable thinking on the underlying LLM. Default to True.
-            cdp_url (Optional[str], optional): CDP URL to connect to an existing browser session.
+            return_screenshots (bool, optional): whether to return screenshots of the agent's states at every step. Default to False.
 
         Returns:
             Union[AgentOutput, Generator[RunAgentResponseChunk, None, None]]: agent output or a generator of response chunks
@@ -140,6 +145,7 @@ class Agent(BaseResource):
             # https://aws.amazon.com/blogs/networking-and-content-delivery/introducing-nlb-tcp-configurable-idle-timeout/
             stream=True,
             enable_thinking=enable_thinking,
+            return_screenshots=return_screenshots,
         )
 
         # For streaming case, use a generator function

lmnr/sdk/client/synchronous/sync_client.py

@@ -150,6 +150,14 @@ class LaminarClient:
         if hasattr(self, "_client"):
             self.__client.close()
 
+    @property
+    def base_url(self) -> str:
+        return self.__base_url
+
+    @property
+    def project_api_key(self) -> str:
+        return self.__project_api_key
+
     def __enter__(self: _T) -> _T:
         return self
 

lmnr/sdk/evaluations.py

@@ -35,15 +35,13 @@ MAX_EXPORT_BATCH_SIZE = 64
 def get_evaluation_url(
     project_id: str, evaluation_id: str, base_url: Optional[str] = None
 ):
-    if not base_url:
+    if not base_url or base_url == "https://api.lmnr.ai":
         base_url = "https://www.lmnr.ai"
 
     url = base_url
-    if url.endswith("/"):
-        url = url[:-1]
+    url = re.sub(r"\/$", "", url)
     if url.endswith("localhost") or url.endswith("127.0.0.1"):
-        # We best effort assume that the frontend is running on port 3000
-        # TODO: expose the frontend port?
+        # We best effort assume that the frontend is running on port 5667
         url = url + ":5667"
     return f"{url}/project/{project_id}/evaluations/{evaluation_id}"
 
@@ -408,8 +406,8 @@ def evaluate(
 
     If there is no event loop, creates it and runs the evaluation until
     completion.
-    If there is an event loop, schedules the evaluation as a task in the
-    event loop and returns an awaitable handle.
+    If there is an event loop, returns an awaitable handle immediately. IMPORTANT:
+    You must await the call to `evaluate`.
 
     Parameters:
         data (Union[list[EvaluationDatapoint|dict]], EvaluationDataset]):\
@@ -482,6 +480,6 @@ def evaluate(
     else:
         loop = asyncio.get_event_loop()
         if loop.is_running():
-            return loop.run_until_complete(evaluation.run())
+            return evaluation.run()
         else:
             return asyncio.run(evaluation.run())

lmnr/sdk/laminar.py

@@ -2,7 +2,7 @@ from contextlib import contextmanager
 from contextvars import Context
 from lmnr.openllmetry_sdk import TracerManager
 from lmnr.openllmetry_sdk.instruments import Instruments
-from lmnr.openllmetry_sdk.tracing import get_tracer
+from lmnr.openllmetry_sdk import get_tracer
 from lmnr.openllmetry_sdk.tracing.attributes import (
     ASSOCIATION_PROPERTIES,
     Attributes,
@@ -638,9 +638,22 @@ class Laminar:
     ) -> LaminarSpanContext:
         return LaminarSpanContext.deserialize(span_context)
 
+    @classmethod
+    def flush(cls) -> bool:
+        """Flush the internal tracer.
+
+        Returns:
+            bool: True if the tracer was flushed, False otherwise
+            (e.g. no tracer or timeout).
+        """
+        if not cls.is_initialized():
+            return False
+        return TracerManager.flush()
+
     @classmethod
     def shutdown(cls):
-        TracerManager.flush()
+        cls.__initialized = False
+        return TracerManager.shutdown()
 
     @classmethod
     def set_session(