lmnr 0.4.55__tar.gz → 0.4.57__tar.gz

{lmnr-0.4.55 → lmnr-0.4.57}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lmnr
-Version: 0.4.55
+Version: 0.4.57
 Summary: Python SDK for Laminar
 License: Apache-2.0
 Author: lmnr.ai
@@ -41,7 +41,6 @@ Provides-Extra: watsonx
 Provides-Extra: weaviate
 Requires-Dist: aiohttp (>=3.0)
 Requires-Dist: argparse (>=1.0)
-Requires-Dist: deprecated (>=1.0)
 Requires-Dist: grpcio (<1.68.0)
 Requires-Dist: opentelemetry-api (>=1.28.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.28.0)

{lmnr-0.4.55 → lmnr-0.4.57}/pyproject.toml

@@ -6,7 +6,7 @@
 
 [project]
 name = "lmnr"
-version = "0.4.55"
+version = "0.4.57"
 description = "Python SDK for Laminar"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -27,7 +27,6 @@ dependencies = [
   "opentelemetry-instrumentation-urllib3 (>=0.50b0)",
   "opentelemetry-instrumentation-threading (>=0.50b0)",
   "opentelemetry-semantic-conventions-ai (>=0.4.2)",
-  "deprecated (>=1.0)",
   "tqdm (>=4.0)",
   "argparse (>=1.0)",
   "aiohttp (>=3.0)",

{lmnr-0.4.55 → lmnr-0.4.57}/src/lmnr/openllmetry_sdk/__init__.py

@@ -32,6 +32,9 @@ class Traceloop:
         should_enrich_metrics: bool = False,
         resource_attributes: dict = {},
         instruments: Optional[Set[Instruments]] = None,
+        base_http_url: Optional[str] = None,
+        project_api_key: Optional[str] = None,
+        max_export_batch_size: Optional[int] = None,
     ) -> None:
         if not is_tracing_enabled():
             return
@@ -69,4 +72,7 @@ class Traceloop:
             exporter=exporter,
             should_enrich_metrics=should_enrich_metrics,
             instruments=instruments,
+            base_http_url=base_http_url,
+            project_api_key=project_api_key,
+            max_export_batch_size=max_export_batch_size,
         )

{lmnr-0.4.55 → lmnr-0.4.57}/src/lmnr/openllmetry_sdk/instruments.py

@@ -21,6 +21,7 @@ class Instruments(Enum):
     OLLAMA = "ollama"
     OPENAI = "openai"
     PINECONE = "pinecone"
+    PLAYWRIGHT = "playwright"
     QDRANT = "qdrant"
     REPLICATE = "replicate"
     SAGEMAKER = "sagemaker"

{lmnr-0.4.55 → lmnr-0.4.57}/src/lmnr/openllmetry_sdk/tracing/tracing.py

@@ -6,6 +6,7 @@ import logging
 from contextvars import Context
 from lmnr.sdk.log import VerboseColorfulFormatter
 from lmnr.openllmetry_sdk.instruments import Instruments
+from lmnr.sdk.browser import init_browser_tracing
 from lmnr.openllmetry_sdk.tracing.attributes import (
     ASSOCIATION_PROPERTIES,
     SPAN_INSTRUMENTATION_SOURCE,
@@ -80,6 +81,9 @@ class TracerWrapper(object):
         exporter: Optional[SpanExporter] = None,
         should_enrich_metrics: bool = False,
         instruments: Optional[Set[Instruments]] = None,
+        base_http_url: Optional[str] = None,
+        project_api_key: Optional[str] = None,
+        max_export_batch_size: Optional[int] = None,
     ) -> "TracerWrapper":
         cls._initialize_logger(cls)
         if not hasattr(cls, "instance"):
@@ -106,7 +110,8 @@ class TracerWrapper(object):
                     )
                 else:
                     obj.__spans_processor: SpanProcessor = BatchSpanProcessor(
-                        obj.__spans_exporter
+                        obj.__spans_exporter,
+                        max_export_batch_size=max_export_batch_size,
                     )
                 obj.__spans_processor_original_on_start = None
 
@@ -122,6 +127,8 @@ class TracerWrapper(object):
             instrument_set = init_instrumentations(
                 should_enrich_metrics,
                 instruments,
+                base_http_url=base_http_url,
+                project_api_key=project_api_key,
             )
 
             if not instrument_set:
@@ -286,6 +293,8 @@ def init_instrumentations(
     should_enrich_metrics: bool,
     instruments: Optional[Set[Instruments]] = None,
     block_instruments: Optional[Set[Instruments]] = None,
+    base_http_url: Optional[str] = None,
+    project_api_key: Optional[str] = None,
 ):
     block_instruments = block_instruments or set()
     # These libraries are not instrumented by default,
@@ -397,6 +406,9 @@ def init_instrumentations(
         elif instrument == Instruments.WEAVIATE:
             if init_weaviate_instrumentor():
                 instrument_set = True
+        elif instrument == Instruments.PLAYWRIGHT:
+            if init_browser_tracing(base_http_url, project_api_key):
+                instrument_set = True
         else:
             module_logger.warning(
                 f"Warning: {instrument} instrumentation does not exist."

lmnr-0.4.57/src/lmnr/sdk/browser/__init__.py

@@ -0,0 +1,9 @@
+from lmnr.openllmetry_sdk.utils.package_check import is_package_installed
+
+
+def init_browser_tracing(http_url: str, project_api_key: str):
+    if is_package_installed("playwright"):
+        from .playwright_patch import init_playwright_tracing
+
+        init_playwright_tracing(http_url, project_api_key)
+    # Other browsers can be added here

lmnr-0.4.57/src/lmnr/sdk/browser/playwright_patch.py

@@ -0,0 +1,249 @@
+import opentelemetry
+import uuid
+import asyncio
+
+try:
+    from playwright.async_api import BrowserContext, Page
+    from playwright.sync_api import (
+        BrowserContext as SyncBrowserContext,
+        Page as SyncPage,
+    )
+except ImportError as e:
+    raise ImportError(
+        f"Attempted to import {__file__}, but it is designed "
+        "to patch Playwright, which is not installed. Use `pip install playwright` "
+        "to install Playwright or remove this import."
+    ) from e
+
+_original_new_page = None
+_original_new_page_async = None
+
+INJECT_PLACEHOLDER = """
+([baseUrl, projectApiKey]) => {
+    const serverUrl = `${baseUrl}/v1/browser-sessions/events`;
+    const FLUSH_INTERVAL = 1000;
+    const HEARTBEAT_INTERVAL = 1000;
+
+    window.rrwebEventsBatch = [];
+    
+    window.sendBatch = async () => {
+        if (window.rrwebEventsBatch.length === 0) return;
+        
+        const eventsPayload = {
+            sessionId: window.rrwebSessionId,
+            traceId: window.traceId,
+            events: window.rrwebEventsBatch
+        };
+        
+        try {
+            const jsonString = JSON.stringify(eventsPayload);
+            const uint8Array = new TextEncoder().encode(jsonString);
+            
+            const cs = new CompressionStream('gzip');
+            const compressedStream = await new Response(
+                new Response(uint8Array).body.pipeThrough(cs)
+            ).arrayBuffer();
+            
+            const compressedArray = new Uint8Array(compressedStream);
+            
+            const blob = new Blob([compressedArray], { type: 'application/octet-stream' });
+            
+            const response = await fetch(serverUrl, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'Content-Encoding': 'gzip',
+                    'Authorization': `Bearer ${projectApiKey}`
+                },
+                body: blob,
+                compress: false,
+                credentials: 'omit',
+                mode: 'cors',
+                cache: 'no-cache',
+            });
+            
+            if (!response.ok) {
+                throw new Error(`HTTP error! status: ${response.status}`);
+            }
+            
+            window.rrwebEventsBatch = [];
+        } catch (error) {
+            console.error('Failed to send events:', error);
+        }
+    };
+
+    setInterval(() => window.sendBatch(), FLUSH_INTERVAL);
+
+    setInterval(() => {
+        window.rrwebEventsBatch.push({
+            type: 6,
+            data: { source: 'heartbeat' },
+            timestamp: Date.now()
+        });
+    }, HEARTBEAT_INTERVAL);
+
+    window.rrweb.record({
+        emit(event) {
+            window.rrwebEventsBatch.push(event);            
+        }
+    });
+
+    window.addEventListener('beforeunload', () => {
+        window.sendBatch();
+    });
+}
+"""
+
+
+def init_playwright_tracing(http_url: str, project_api_key: str):
+
+    def inject_rrweb(page: SyncPage):
+        # Get current trace ID from active span
+        current_span = opentelemetry.trace.get_current_span()
+        current_span.set_attribute("lmnr.internal.has_browser_session", True)
+        trace_id = format(current_span.get_span_context().trace_id, "032x")
+        session_id = str(uuid.uuid4().hex)
+
+        # Generate UUID session ID and set trace ID
+        page.evaluate(
+            """([traceId, sessionId]) => {
+            window.rrwebSessionId = sessionId;
+            window.traceId = traceId;
+        }""",
+            [trace_id, session_id],
+        )
+
+        # Load rrweb from CDN
+        page.add_script_tag(
+            url="https://cdn.jsdelivr.net/npm/rrweb@latest/dist/rrweb.min.js"
+        )
+
+        # Update the recording setup to include trace ID
+        page.evaluate(
+            INJECT_PLACEHOLDER,
+            [http_url, project_api_key],
+        )
+
+    async def inject_rrweb_async(page: Page):
+        try:
+            # Wait for the page to be in a ready state first
+            await page.wait_for_load_state("domcontentloaded")
+
+            # Get current trace ID from active span
+            current_span = opentelemetry.trace.get_current_span()
+            current_span.set_attribute("lmnr.internal.has_browser_session", True)
+            trace_id = format(current_span.get_span_context().trace_id, "032x")
+            session_id = str(uuid.uuid4().hex)
+
+            # Generate UUID session ID and set trace ID
+            await page.evaluate(
+                """([traceId, sessionId]) => {
+                window.rrwebSessionId = sessionId;
+                window.traceId = traceId;
+            }""",
+                [trace_id, session_id],
+            )
+
+            # Load rrweb from CDN
+            await page.add_script_tag(
+                url="https://cdn.jsdelivr.net/npm/rrweb@latest/dist/rrweb.min.js"
+            )
+
+            await page.wait_for_function(
+                """(() => window.rrweb || 'rrweb' in window)"""
+            )
+
+            # Update the recording setup to include trace ID
+            await page.evaluate(
+                INJECT_PLACEHOLDER,
+                [http_url, project_api_key],
+            )
+        except Exception as e:
+            print(f"Error injecting rrweb: {e}")
+
+    def handle_navigation(page: SyncPage):
+        def on_load():
+            inject_rrweb(page)
+
+        page.on("load", on_load)
+        inject_rrweb(page)
+
+    async def handle_navigation_async(page: Page):
+        async def on_load():
+            await inject_rrweb_async(page)
+
+        page.on("load", lambda: asyncio.create_task(on_load()))
+        await inject_rrweb_async(page)
+
+    async def patched_new_page_async(self: BrowserContext, *args, **kwargs):
+        # Modify CSP to allow required domains
+        async def handle_route(route):
+            try:
+                response = await route.fetch()
+                headers = dict(response.headers)
+
+                # Find and modify CSP header
+                for header_name in headers:
+                    if header_name.lower() == "content-security-policy":
+                        csp = headers[header_name]
+                        parts = csp.split(";")
+                        for i, part in enumerate(parts):
+                            if "script-src" in part:
+                                parts[i] = f"{part.strip()} cdn.jsdelivr.net"
+                            elif "connect-src" in part:
+                                parts[i] = f"{part.strip()} " + http_url
+                        if not any("connect-src" in part for part in parts):
+                            parts.append(" connect-src 'self' " + http_url)
+                        headers[header_name] = ";".join(parts)
+
+                await route.fulfill(response=response, headers=headers)
+            except Exception:
+                await route.continue_()
+
+        await self.route("**/*", handle_route)
+        page = await _original_new_page_async(self, *args, **kwargs)
+        await handle_navigation_async(page)
+        return page
+
+    def patched_new_page(self: SyncBrowserContext, *args, **kwargs):
+        # Modify CSP to allow required domains
+        def handle_route(route):
+            try:
+                response = route.fetch()
+                headers = dict(response.headers)
+
+                # Find and modify CSP header
+                for header_name in headers:
+                    if header_name.lower() == "content-security-policy":
+                        csp = headers[header_name]
+                        parts = csp.split(";")
+                        for i, part in enumerate(parts):
+                            if "script-src" in part:
+                                parts[i] = f"{part.strip()} cdn.jsdelivr.net"
+                            elif "connect-src" in part:
+                                parts[i] = f"{part.strip()} " + http_url
+                        if not any("connect-src" in part for part in parts):
+                            parts.append(" connect-src 'self' " + http_url)
+                        headers[header_name] = ";".join(parts)
+
+                route.fulfill(response=response, headers=headers)
+            except Exception:
+                # Continue with the original request without modification
+                route.continue_()
+
+        self.route("**/*", handle_route)
+        page = _original_new_page(self, *args, **kwargs)
+        handle_navigation(page)
+        return page
+
+    def patch_browser():
+        global _original_new_page, _original_new_page_async
+        if _original_new_page_async is None:
+            _original_new_page_async = BrowserContext.new_page
+            BrowserContext.new_page = patched_new_page_async
+
+        if _original_new_page is None:
+            _original_new_page = SyncBrowserContext.new_page
+            SyncBrowserContext.new_page = patched_new_page
+
+    patch_browser()

{lmnr-0.4.55 → lmnr-0.4.57}/src/lmnr/sdk/evaluations.py

@@ -2,7 +2,6 @@ import asyncio
 import re
 import sys
 import uuid
-
 from tqdm import tqdm
 from typing import Any, Awaitable, Optional, Set, Union
 
@@ -27,9 +26,12 @@ from .types import (
 from .utils import is_async
 
 DEFAULT_BATCH_SIZE = 5
+MAX_EXPORT_BATCH_SIZE = 64
 
 
-def get_evaluation_url(project_id: str, evaluation_id: str, base_url: Optional[str] = None):
+def get_evaluation_url(
+    project_id: str, evaluation_id: str, base_url: Optional[str] = None
+):
     if not base_url:
         base_url = "https://www.lmnr.ai"
 
@@ -39,7 +41,7 @@ def get_evaluation_url(project_id: str, evaluation_id: str, base_url: Optional[s
     if url.endswith("localhost") or url.endswith("127.0.0.1"):
         # We best effort assume that the frontend is running on port 3000
         # TODO: expose the frontend port?
-        url = url + ":3000"
+        url = url + ":5667"
     return f"{url}/project/{project_id}/evaluations/{evaluation_id}"
 
 
@@ -97,13 +99,14 @@ class Evaluation:
         evaluators: dict[str, EvaluatorFunction],
         human_evaluators: list[HumanEvaluator] = [],
         name: Optional[str] = None,
-        group_id: Optional[str] = None,
-        batch_size: int = DEFAULT_BATCH_SIZE,
+        group_name: Optional[str] = None,
+        concurrency_limit: int = DEFAULT_BATCH_SIZE,
         project_api_key: Optional[str] = None,
         base_url: Optional[str] = None,
         http_port: Optional[int] = None,
         grpc_port: Optional[int] = None,
         instruments: Optional[Set[Instruments]] = None,
+        max_export_batch_size: Optional[int] = MAX_EXPORT_BATCH_SIZE,
     ):
         """
         Initializes an instance of the Evaluations class.
@@ -131,12 +134,12 @@ class Evaluation:
                 Used to identify the evaluation in the group.\
                 If not provided, a random name will be generated.
                 Defaults to None.
-            group_id (Optional[str], optional): an identifier to group\
-                evaluations. Only evaluations within the same group_id can be\
+            group_name (Optional[str], optional): an identifier to group\
+                evaluations. Only evaluations within the same group_name can be\
                 visually compared. If not provided, "default" is assigned.
                 Defaults to None
-            batch_size (int, optional): The batch size for evaluation. This many\
-                data points will be evaluated in parallel.
+            concurrency_limit (int, optional): The concurrency limit for evaluation. This many\
+                data points will be evaluated in parallel with a pool of workers.
                 Defaults to DEFAULT_BATCH_SIZE.
             project_api_key (Optional[str], optional): The project API key.\
                 If not provided, LMNR_PROJECT_API_KEY environment variable is\
@@ -180,17 +183,20 @@ class Evaluation:
             self.data = data
         self.executor = executor
         self.evaluators = evaluators
-        self.group_id = group_id
+        self.group_name = group_name
         self.name = name
-        self.batch_size = batch_size
+        self.concurrency_limit = concurrency_limit
+        self.batch_size = concurrency_limit
         self._logger = get_default_logger(self.__class__.__name__)
         self.human_evaluators = human_evaluators
+        self.upload_tasks = []  # Add this line to track upload tasks
         L.initialize(
             project_api_key=project_api_key,
             base_url=base_url,
             http_port=http_port,
             grpc_port=grpc_port,
             instruments=instruments,
+            max_export_batch_size=max_export_batch_size,
         )
 
     async def run(self) -> Awaitable[None]:
@@ -200,49 +206,60 @@ class Evaluation:
 
     async def _run(self) -> None:
         self.reporter.start(len(self.data))
-
         try:
-            result_datapoints = await self._evaluate_in_batches()
+            evaluation = await L.init_eval(name=self.name, group_name=self.group_name)
+            result_datapoints = await self._evaluate_in_batches(evaluation.id)
+
+            # Wait for all background upload tasks to complete
+            if self.upload_tasks:
+                self._logger.debug(
+                    f"Waiting for {len(self.upload_tasks)} upload tasks to complete"
+                )
+                await asyncio.gather(*self.upload_tasks)
+                self._logger.debug("All upload tasks completed")
         except Exception as e:
             self.reporter.stopWithError(e)
             self.is_finished = True
             return
 
-        # For now add all human evaluators to all result datapoints
-        # In the future, we will add ways to specify which human evaluators
-        # to add to which result datapoints, e.g. sample some randomly
         for result_datapoint in result_datapoints:
             result_datapoint.human_evaluators = self.human_evaluators or {}
 
-        evaluation = await L.create_evaluation(
-            data=result_datapoints, group_id=self.group_id, name=self.name
-        )
         average_scores = get_average_scores(result_datapoints)
         self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
         self.is_finished = True
 
-    async def _evaluate_in_batches(self) -> list[EvaluationResultDatapoint]:
-        result_datapoints = []
-        for i in range(0, len(self.data), self.batch_size):
-            batch = (
-                self.data[i : i + self.batch_size]
-                if isinstance(self.data, list)
-                else self.data.slice(i, i + self.batch_size)
-            )
-            batch_datapoints = await self._evaluate_batch(batch)
-            result_datapoints.extend(batch_datapoints)
-            self.reporter.update(len(batch))
-        return result_datapoints
-
-    async def _evaluate_batch(
-        self, batch: list[Datapoint]
+    async def _evaluate_in_batches(
+        self, eval_id: uuid.UUID
     ) -> list[EvaluationResultDatapoint]:
-        batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
-        results = await asyncio.gather(*batch_promises)
-        return results
+
+        semaphore = asyncio.Semaphore(self.concurrency_limit)
+        tasks = []
+        data_iter = self.data if isinstance(self.data, list) else range(len(self.data))
+
+        async def evaluate_task(datapoint, index):
+            try:
+                result = await self._evaluate_datapoint(eval_id, datapoint, index)
+                self.reporter.update(1)
+                return index, result
+            finally:
+                semaphore.release()
+
+        # Create tasks only after acquiring semaphore
+        for idx, item in enumerate(data_iter):
+            await semaphore.acquire()
+            datapoint = item if isinstance(self.data, list) else self.data[item]
+            task = asyncio.create_task(evaluate_task(datapoint, idx))
+            tasks.append(task)
+
+        # Wait for all tasks to complete and preserve order
+        results = await asyncio.gather(*tasks)
+        ordered_results = [result for _, result in sorted(results, key=lambda x: x[0])]
+
+        return ordered_results
 
     async def _evaluate_datapoint(
-        self, datapoint: Datapoint
+        self, eval_id: uuid.UUID, datapoint: Datapoint, index: int
     ) -> EvaluationResultDatapoint:
         with L.start_as_current_span("evaluation") as evaluation_span:
             L._set_trace_type(trace_type=TraceType.EVALUATION)
@@ -251,11 +268,15 @@ class Evaluation:
                 "executor", input={"data": datapoint.data}
             ) as executor_span:
                 executor_span.set_attribute(SPAN_TYPE, SpanType.EXECUTOR.value)
-                output = (
-                    await self.executor(datapoint.data)
-                    if is_async(self.executor)
-                    else self.executor(datapoint.data)
-                )
+                # Run synchronous executors in a thread pool to avoid blocking
+                if not is_async(self.executor):
+                    loop = asyncio.get_event_loop()
+                    output = await loop.run_in_executor(
+                        None, self.executor, datapoint.data
+                    )
+                else:
+                    output = await self.executor(datapoint.data)
+
                 L.set_span_output(output)
                 executor_span_id = uuid.UUID(
                     int=executor_span.get_span_context().span_id
@@ -283,14 +304,28 @@ class Evaluation:
                     scores.update(value)
 
             trace_id = uuid.UUID(int=evaluation_span.get_span_context().trace_id)
-            return EvaluationResultDatapoint(
-                data=datapoint.data,
-                target=target,
-                executor_output=output,
-                scores=scores,
-                trace_id=trace_id,
-                executor_span_id=executor_span_id,
-            )
+
+        datapoint = EvaluationResultDatapoint(
+            data=datapoint.data,
+            target=target,
+            executor_output=output,
+            scores=scores,
+            trace_id=trace_id,
+            # For now add all human evaluators to all result datapoints
+            # In the future, we will add ways to specify which human evaluators
+            # to add to which result datapoints, e.g. sample some randomly
+            human_evaluators=self.human_evaluators,
+            executor_span_id=executor_span_id,
+            index=index,
+        )
+
+        # Create background upload task without awaiting it
+        upload_task = asyncio.create_task(
+            L.save_eval_datapoints(eval_id, [datapoint], self.group_name)
+        )
+        self.upload_tasks.append(upload_task)
+
+        return datapoint
 
 
 def evaluate(
@@ -299,8 +334,9 @@ def evaluate(
     evaluators: dict[str, EvaluatorFunction],
     human_evaluators: list[HumanEvaluator] = [],
     name: Optional[str] = None,
-    group_id: Optional[str] = None,
-    batch_size: int = DEFAULT_BATCH_SIZE,
+    group_id: Optional[str] = None,  # Deprecated
+    group_name: Optional[str] = None,
+    concurrency_limit: int = DEFAULT_BATCH_SIZE,
     project_api_key: Optional[str] = None,
     base_url: Optional[str] = None,
     http_port: Optional[int] = None,
@@ -318,12 +354,12 @@ def evaluate(
 
     Parameters:
         data (Union[list[EvaluationDatapoint|dict]], EvaluationDataset]):\
-                    List of data points to evaluate or an evaluation dataset.
-                        `data` is the input to the executor function,
-                        `target` is the input to the evaluator function.
+            List of data points to evaluate or an evaluation dataset.
+                `data` is the input to the executor function,
+                `target` is the input to the evaluator function.
         executor (Callable[..., Any]): The executor function.\
-                        Takes the data point + any additional arguments\
-                        and returns the output to evaluate.
+            Takes the data point + any additional arguments\
+            and returns the output to evaluate.
         evaluators (List[Callable[..., Any]]): 
             evaluators (dict[str, Callable[..., Any]]): Evaluator functions and\
                 names. Each evaluator function takes the output of the executor\
@@ -337,14 +373,19 @@ def evaluate(
             evaluator only holds the queue name.
             Defaults to an empty list.
         name (Optional[str], optional): Optional name of the evaluation.\
-                        Used to identify the evaluation in the group.\
-                        If not provided, a random name will be generated.
-                        Defaults to None.
-        group_id (Optional[str], optional): an identifier to group evaluations.\
+            Used to identify the evaluation in the group. If not provided, a\
+            random name will be generated.
+            Defaults to None.
+        group_id (Optional[str], optional): [DEPRECATED] Use group_name instead.
+                        An identifier to group evaluations.\
                         Only evaluations within the same group_id can be\
                         visually compared. If not provided, set to "default".
                         Defaults to None
-        batch_size (int, optional): The batch size for evaluation.
+        group_name (Optional[str], optional): An identifier to group evaluations.\
+            Only evaluations within the same group_name can be visually compared.\
+            If not provided, set to "default".
+            Defaults to None
+        concurrency_limit (int, optional): The concurrency limit for evaluation.
                         Defaults to DEFAULT_BATCH_SIZE.
         project_api_key (Optional[str], optional): The project API key.
                         Defaults to None.
@@ -363,15 +404,19 @@ def evaluate(
                         will be used.
                         Defaults to None.
     """
+    if group_id:
+        raise DeprecationWarning("group_id is deprecated. Use group_name instead.")
+
+    group_name = group_name or group_id
 
     evaluation = Evaluation(
         data=data,
         executor=executor,
         evaluators=evaluators,
-        group_id=group_id,
+        group_name=group_name,
         human_evaluators=human_evaluators,
         name=name,
-        batch_size=batch_size,
+        concurrency_limit=concurrency_limit,
         project_api_key=project_api_key,
         base_url=base_url,
         http_port=http_port,

{lmnr-0.4.55 → lmnr-0.4.57}/src/lmnr/sdk/laminar.py

@@ -47,7 +47,8 @@ from lmnr.openllmetry_sdk.tracing.tracing import (
 from .log import VerboseColorfulFormatter
 
 from .types import (
-    CreateEvaluationResponse,
+    HumanEvaluator,
+    InitEvaluationResponse,
     EvaluationResultDatapoint,
     GetDatapointsResponse,
     PipelineRunError,
@@ -78,6 +79,7 @@ class Laminar:
         grpc_port: Optional[int] = None,
         instruments: Optional[Set[Instruments]] = None,
         disable_batch: bool = False,
+        max_export_batch_size: Optional[int] = None,
     ):
         """Initialize Laminar context across the application.
         This method must be called before using any other Laminar methods or
@@ -142,12 +144,15 @@ class Laminar:
         cls._initialize_logger()
 
         Traceloop.init(
+            base_http_url=cls.__base_http_url,
+            project_api_key=cls.__project_api_key,
             exporter=OTLPSpanExporter(
                 endpoint=cls.__base_grpc_url,
                 headers={"authorization": f"Bearer {cls.__project_api_key}"},
             ),
             instruments=instruments,
             disable_batch=disable_batch,
+            max_export_batch_size=max_export_batch_size,
         )
 
     @classmethod
@@ -687,33 +692,44 @@ class Laminar:
         set_association_properties(props)
 
     @classmethod
-    async def create_evaluation(
-        cls,
-        data: list[EvaluationResultDatapoint],
-        group_id: Optional[str] = None,
-        name: Optional[str] = None,
-    ) -> CreateEvaluationResponse:
+    async def init_eval(
+        cls, name: Optional[str] = None, group_name: Optional[str] = None
+    ) -> InitEvaluationResponse:
         async with aiohttp.ClientSession() as session:
             async with session.post(
-                cls.__base_http_url + "/v1/evaluations",
+                cls.__base_http_url + "/v1/evals",
                 json={
-                    "groupId": group_id,
                     "name": name,
-                    "points": [datapoint.to_dict() for datapoint in data],
+                    "groupName": group_name,
                 },
                 headers=cls._headers(),
             ) as response:
-                if response.status != 200:
-                    try:
-                        resp_json = await response.json()
-                        raise ValueError(
-                            f"Error creating evaluation {json.dumps(resp_json)}"
-                        )
-                    except aiohttp.ClientError:
-                        text = await response.text()
-                        raise ValueError(f"Error creating evaluation {text}")
                 resp_json = await response.json()
-                return CreateEvaluationResponse.model_validate(resp_json)
+                return InitEvaluationResponse.model_validate(resp_json)
+
+    @classmethod
+    async def save_eval_datapoints(
+        cls,
+        eval_id: uuid.UUID,
+        datapoints: list[EvaluationResultDatapoint],
+        groupName: Optional[str] = None,
+        human_evaluators: Optional[list[HumanEvaluator]] = None,
+    ):
+        async with aiohttp.ClientSession() as session:
+
+            async with session.post(
+                cls.__base_http_url + f"/v1/evals/{eval_id}/datapoints",
+                json={
+                    "points": [datapoint.to_dict() for datapoint in datapoints],
+                    "groupName": groupName,
+                    "humanEvaluators": human_evaluators,
+                },
+                headers=cls._headers(),
+            ) as response:
+                if response.status != 200:
+                    raise ValueError(
+                        f"Error saving evaluation datapoints: {response.text}"
+                    )
 
     @classmethod
     def get_datapoints(

{lmnr-0.4.55 → lmnr-0.4.57}/src/lmnr/sdk/types.py

@@ -141,11 +141,8 @@ EvaluatorFunction = Callable[
 class HumanEvaluator(pydantic.BaseModel):
     queueName: str
 
-    def __init__(self, queue_name: str):
-        super().__init__(queueName=queue_name)
 
-
-class CreateEvaluationResponse(pydantic.BaseModel):
+class InitEvaluationResponse(pydantic.BaseModel):
     id: uuid.UUID
     createdAt: datetime.datetime
     groupId: str
@@ -161,6 +158,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     human_evaluators: list[HumanEvaluator] = pydantic.Field(default_factory=list)
     trace_id: uuid.UUID
     executor_span_id: uuid.UUID
+    index: int
 
     # uuid is not serializable by default, so we need to convert it to a string
     def to_dict(self):
@@ -180,6 +178,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
                     for v in self.human_evaluators
                 ],
                 "executorSpanId": str(self.executor_span_id),
+                "index": self.index,
             }
         except Exception as e:
             raise ValueError(f"Error serializing EvaluationResultDatapoint: {e}")