lmnr 0.6.1__tar.gz → 0.6.3__tar.gz

{lmnr-0.6.1 → lmnr-0.6.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lmnr
-Version: 0.6.1
+Version: 0.6.3
 Summary: Python SDK for Laminar
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.6.1 → lmnr-0.6.3}/pyproject.toml

@@ -6,7 +6,7 @@
 
 [project]
 name = "lmnr"
-version = "0.6.1"
+version = "0.6.3"
 description = "Python SDK for Laminar"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/cli.py

@@ -8,7 +8,7 @@ from typing import Optional
 
 from lmnr.sdk.evaluations import Evaluation
 
-from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
+from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCES
 from .sdk.log import get_default_logger
 
 LOG = get_default_logger(__name__)
@@ -38,10 +38,10 @@ async def run_evaluation(args):
     else:
         files = [args.file]
 
-    for file in files:
-        prep_token = PREPARE_ONLY.set(True)
-        LOG.info(f"Running evaluation from {file}")
-        try:
+    prep_token = PREPARE_ONLY.set(True)
+    try:
+        for file in files:
+            LOG.info(f"Running evaluation from {file}")
             file = os.path.abspath(file)
             name = "user_module" + file
 
@@ -55,16 +55,24 @@ async def run_evaluation(args):
             sys.modules[name] = mod
 
             spec.loader.exec_module(mod)
-            evaluation: Optional[Evaluation] = EVALUATION_INSTANCE.get()
-            if evaluation is None:
+            evaluations: Optional[list[Evaluation]] = EVALUATION_INSTANCES.get()
+            if evaluations is None:
                 LOG.warning("Evaluation instance not found")
                 if args.fail_on_error:
                     return
                 continue
 
-            await evaluation.run()
-        finally:
-            PREPARE_ONLY.reset(prep_token)
+            LOG.info(f"Loaded {len(evaluations)} evaluations from {file}")
+
+            for evaluation in evaluations:
+                try:
+                    await evaluation.run()
+                except Exception as e:
+                    LOG.error(f"Error running evaluation: {e}")
+                    if args.fail_on_error:
+                        raise
+    finally:
+        PREPARE_ONLY.reset(prep_token)
 
 
 def cli():

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/opentelemetry_lib/decorators/__init__.py

@@ -9,6 +9,10 @@ from opentelemetry import trace
 from opentelemetry import context as context_api
 from opentelemetry.trace import Span
 
+from lmnr.opentelemetry_lib.tracing.context_properties import (
+    remove_association_properties,
+    update_association_properties,
+)
 from lmnr.sdk.utils import get_input_from_func_args, is_method
 from lmnr.opentelemetry_lib import MAX_MANUAL_SPAN_PAYLOAD_SIZE
 from lmnr.opentelemetry_lib.tracing.tracer import get_tracer
@@ -42,6 +46,7 @@ def entity_method(
     ignore_inputs: Optional[list[str]] = None,
     ignore_output: bool = False,
     span_type: Union[Literal["DEFAULT"], Literal["LLM"], Literal["TOOL"]] = "DEFAULT",
+    association_properties: Optional[dict[str, Any]] = None,
 ):
     def decorate(fn):
         @wraps(fn)
@@ -51,6 +56,9 @@ def entity_method(
 
             span_name = name or fn.__name__
 
+            if association_properties is not None:
+                update_association_properties(association_properties)
+
             with get_tracer() as tracer:
                 span = tracer.start_span(span_name, attributes={SPAN_TYPE: span_type})
 
@@ -111,7 +119,8 @@ def entity_method(
 
                 span.end()
                 context_api.detach(ctx_token)
-
+                if association_properties is not None:
+                    remove_association_properties(association_properties)
                 return res
 
         return wrap
@@ -126,6 +135,7 @@ def aentity_method(
     ignore_inputs: Optional[list[str]] = None,
     ignore_output: bool = False,
     span_type: Union[Literal["DEFAULT"], Literal["LLM"], Literal["TOOL"]] = "DEFAULT",
+    association_properties: Optional[dict[str, Any]] = None,
 ):
     def decorate(fn):
         @wraps(fn)
@@ -135,6 +145,9 @@ def aentity_method(
 
             span_name = name or fn.__name__
 
+            if association_properties is not None:
+                update_association_properties(association_properties)
+
             with get_tracer() as tracer:
                 span = tracer.start_span(span_name, attributes={SPAN_TYPE: span_type})
 
@@ -187,6 +200,8 @@ def aentity_method(
                     pass
 
                 span.end()
+                if association_properties is not None:
+                    remove_association_properties(association_properties)
                 context_api.detach(ctx_token)
 
                 return res

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/browser/browser_use_otel.py

@@ -83,12 +83,13 @@ async def _wrap(tracer: Tracer, to_wrap, wrapped, instance, args, kwargs):
         span.set_attributes(attributes)
         result = await wrapped(*args, **kwargs)
         if not to_wrap.get("ignore_output"):
+            to_serialize = result
             if isinstance(result, AgentHistoryList):
-                result = result.final_result()
+                to_serialize = result.final_result()
             serialized = (
-                result.model_dump_json()
-                if isinstance(result, pydantic.BaseModel)
-                else json_dumps(result)
+                to_serialize.model_dump_json()
+                if isinstance(to_serialize, pydantic.BaseModel)
+                else json_dumps(to_serialize)
             )
             span.set_attribute("lmnr.span.output", serialized)
         return result

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/client/asynchronous/resources/evals.py

@@ -1,12 +1,13 @@
 """Evals resource for interacting with Laminar evaluations API."""
 
 import uuid
-from typing import Optional
+from typing import Optional, Union
 
 from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
 from lmnr.sdk.types import (
     InitEvaluationResponse,
     EvaluationResultDatapoint,
+    PartialEvaluationDatapoint,
 )
 
 
@@ -39,14 +40,14 @@ class AsyncEvals(BaseAsyncResource):
     async def save_datapoints(
         self,
         eval_id: uuid.UUID,
-        datapoints: list[EvaluationResultDatapoint],
+        datapoints: list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]],
         group_name: Optional[str] = None,
     ):
         """Save evaluation datapoints.
 
         Args:
             eval_id (uuid.UUID): The evaluation ID.
-            datapoints (list[EvaluationResultDatapoint]): The datapoints to save.
+            datapoints (list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]]): The datapoints to save.
             group_name (Optional[str], optional): Group name for the datapoints. Defaults to None.
 
         Raises:

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/client/synchronous/resources/evals.py

@@ -2,12 +2,13 @@
 
 import uuid
 import urllib.parse
-from typing import Optional
+from typing import Optional, Union
 
 from lmnr.sdk.client.synchronous.resources.base import BaseResource
 from lmnr.sdk.types import (
     InitEvaluationResponse,
     EvaluationResultDatapoint,
+    PartialEvaluationDatapoint,
     GetDatapointsResponse,
 )
 
@@ -41,14 +42,14 @@ class Evals(BaseResource):
     def save_datapoints(
         self,
         eval_id: uuid.UUID,
-        datapoints: list[EvaluationResultDatapoint],
+        datapoints: list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]],
         group_name: Optional[str] = None,
     ):
         """Save evaluation datapoints.
 
         Args:
             eval_id (uuid.UUID): The evaluation ID.
-            datapoints (list[EvaluationResultDatapoint]): The datapoints to save.
+            datapoints (list[Union[EvaluationResultDatapoint, PartialEvaluationDatapoint]]): The datapoints to save.
             group_name (Optional[str], optional): Group name for the datapoints. Defaults to None.
 
         Raises:

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/decorators.py

@@ -1,16 +1,14 @@
 from lmnr.opentelemetry_lib.decorators import (
     entity_method,
     aentity_method,
+    json_dumps,
 )
 from opentelemetry.trace import INVALID_SPAN, get_current_span
 
-from typing import Callable, Literal, Optional, TypeVar, Union, cast
+from typing import Any, Callable, Literal, Optional, TypeVar, Union, cast
 from typing_extensions import ParamSpec
 
 from lmnr.opentelemetry_lib.tracing.attributes import SESSION_ID
-from lmnr.opentelemetry_lib.tracing.context_properties import (
-    update_association_properties,
-)
 
 from .utils import is_async
 
@@ -23,10 +21,12 @@ def observe(
     *,
     name: Optional[str] = None,
     session_id: Optional[str] = None,
+    user_id: Optional[str] = None,
     ignore_input: bool = False,
     ignore_output: bool = False,
     span_type: Union[Literal["DEFAULT"], Literal["LLM"], Literal["TOOL"]] = "DEFAULT",
     ignore_inputs: Optional[list[str]] = None,
+    metadata: Optional[dict[str, Any]] = None,
 ) -> Callable[[Callable[P, R]], Callable[P, R]]:
     """The main decorator entrypoint for Laminar. This is used to wrap
     functions and methods to create spans.
@@ -37,6 +37,9 @@ def observe(
                         Defaults to None.
         session_id (Optional[str], optional): Session ID to associate with the\
                         span and the following context. Defaults to None.
+        user_id (Optional[str], optional): User ID to associate with the\
+                        span and the following context. This is different from \
+                        ID of a Laminar user. Defaults to None.
         ignore_input (bool, optional): Whether to ignore ALL input of the\
                         wrapped function. Defaults to False.
         ignore_output (bool, optional): Whether to ignore ALL output of the\
@@ -49,6 +52,8 @@ def observe(
                         `sensitive_data` argument, you can pass ["sensitive_data"] to\
                         this argument.
                         Defaults to None.
+        metadata (Optional[dict[str, Any]], optional): Metadata to associate with the\
+                        trace. Must be JSON serializable. Defaults to None.
     Raises:
         Exception: re-raises the exception if the wrapped function raises
                    an exception
@@ -65,14 +70,25 @@ def observe(
         association_properties = {}
         if session_id is not None:
             association_properties["session_id"] = session_id
-        update_association_properties(association_properties)
-        return (
+        if user_id is not None:
+            association_properties["user_id"] = user_id
+        if metadata is not None:
+            association_properties.update(
+                {
+                    f"metadata.{k}": (
+                        v if isinstance(v, (str, int, float, bool)) else json_dumps(v)
+                    )
+                    for k, v in metadata.items()
+                }
+            )
+        result = (
             aentity_method(
                 name=name,
                 ignore_input=ignore_input,
                 ignore_output=ignore_output,
                 span_type=span_type,
                 ignore_inputs=ignore_inputs,
+                association_properties=association_properties,
             )(func)
             if is_async(func)
             else entity_method(
@@ -81,7 +97,9 @@ def observe(
                 ignore_output=ignore_output,
                 span_type=span_type,
                 ignore_inputs=ignore_inputs,
+                association_properties=association_properties,
             )(func)
         )
+        return result
 
     return cast(Callable, decorator)

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/eval_control.py

@@ -2,4 +2,4 @@ from contextvars import ContextVar
 
 
 PREPARE_ONLY: ContextVar[bool] = ContextVar("__lmnr_prepare_only", default=False)
-EVALUATION_INSTANCE = ContextVar("__lmnr_evaluation_instance")
+EVALUATION_INSTANCES = ContextVar("__lmnr_evaluation_instances")

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/evaluations.py

@@ -11,7 +11,7 @@ from lmnr.opentelemetry_lib.tracing.attributes import SPAN_TYPE
 from lmnr.sdk.client.asynchronous.async_client import AsyncLaminarClient
 from lmnr.sdk.client.synchronous.sync_client import LaminarClient
 from lmnr.sdk.datasets import EvaluationDataset, LaminarDataset
-from lmnr.sdk.eval_control import EVALUATION_INSTANCE, PREPARE_ONLY
+from lmnr.sdk.eval_control import EVALUATION_INSTANCES, PREPARE_ONLY
 from lmnr.sdk.laminar import Laminar as L
 from lmnr.sdk.log import get_default_logger
 from lmnr.sdk.types import (
@@ -204,6 +204,18 @@ class Evaluation:
             )
         self.project_api_key = api_key
 
+        if L.is_initialized():
+            self.client = AsyncLaminarClient(
+                base_url=L.get_base_http_url(),
+                project_api_key=L.get_project_api_key(),
+            )
+            if project_api_key and project_api_key != L.get_project_api_key():
+                self._logger.warning(
+                    "Project API key is different from the one used to initialize"
+                    " Laminar. Ignoring the project API key passed to the evaluation."
+                )
+            return
+
         self.client = AsyncLaminarClient(
             base_url=self.base_http_url,
             project_api_key=self.project_api_key,
@@ -312,6 +324,7 @@ class Evaluation:
                     index=index,
                     trace_id=trace_id,
                     executor_span_id=executor_span_id,
+                    metadata=datapoint.metadata,
                 )
                 # First, create datapoint with trace_id so that we can show the dp in the UI
                 await self.client._evals.save_datapoints(
@@ -367,6 +380,7 @@ class Evaluation:
             human_evaluators=self.human_evaluators,
             executor_span_id=executor_span_id,
             index=index,
+            metadata=datapoint.metadata,
         )
 
         # Create background upload task without awaiting it
@@ -470,10 +484,16 @@ def evaluate(
     )
 
     if PREPARE_ONLY.get():
-        EVALUATION_INSTANCE.set(evaluation)
+        existing_evaluations = EVALUATION_INSTANCES.get([])
+        new_evaluations = (existing_evaluations or []) + [evaluation]
+        EVALUATION_INSTANCES.set(new_evaluations)
+        return None
     else:
-        loop = asyncio.get_event_loop()
-        if loop.is_running():
-            return evaluation.run()
-        else:
+        try:
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                return evaluation.run()
+            else:
+                return asyncio.run(evaluation.run())
+        except RuntimeError:
             return asyncio.run(evaluation.run())

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/laminar.py

@@ -50,6 +50,7 @@ from .types import (
 class Laminar:
     __project_api_key: Optional[str] = None
     __initialized: bool = False
+    __base_http_url: Optional[str] = None
 
     @classmethod
     def initialize(
@@ -133,6 +134,7 @@ class Laminar:
                 cls.__logger.info(f"Using HTTP port passed as an argument: {http_port}")
 
         cls.__initialized = True
+        cls.__base_http_url = f"{url}:{http_port or 443}"
 
         if not os.getenv("OTEL_ATTRIBUTE_COUNT_LIMIT"):
             # each message is at least 2 attributes: role and content,
@@ -700,6 +702,14 @@ class Laminar:
         props.pop("user_id", None)
         set_association_properties(props)
 
+    @classmethod
+    def get_base_http_url(cls):
+        return cls.__base_http_url
+
+    @classmethod
+    def get_project_api_key(cls):
+        return cls.__project_api_key
+
     @classmethod
     def _headers(cls):
         assert cls.__project_api_key is not None, "Project API key is not set"

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/sdk/types.py

@@ -66,6 +66,7 @@ class PartialEvaluationDatapoint(pydantic.BaseModel):
     index: int
     trace_id: uuid.UUID
     executor_span_id: uuid.UUID
+    metadata: EvaluationDatapointMetadata = pydantic.Field(default=None)
 
     # uuid is not serializable by default, so we need to convert it to a string
     def to_dict(self):
@@ -77,6 +78,7 @@ class PartialEvaluationDatapoint(pydantic.BaseModel):
                 "index": self.index,
                 "traceId": str(self.trace_id),
                 "executorSpanId": str(self.executor_span_id),
+                "metadata": serialize(self.metadata) if self.metadata is not None else None,
             }
         except Exception as e:
             raise ValueError(f"Error serializing PartialEvaluationDatapoint: {e}")
@@ -92,6 +94,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     human_evaluators: list[HumanEvaluator] = pydantic.Field(default_factory=list)
     trace_id: uuid.UUID
     executor_span_id: uuid.UUID
+    metadata: EvaluationDatapointMetadata = pydantic.Field(default=None)
 
     # uuid is not serializable by default, so we need to convert it to a string
     def to_dict(self):
@@ -115,6 +118,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
                 ],
                 "executorSpanId": str(self.executor_span_id),
                 "index": self.index,
+                "metadata": serialize(self.metadata) if self.metadata is not None else None,
             }
         except Exception as e:
             raise ValueError(f"Error serializing EvaluationResultDatapoint: {e}")

{lmnr-0.6.1 → lmnr-0.6.3}/src/lmnr/version.py

@@ -3,7 +3,7 @@ import httpx
 from packaging import version
 
 
-__version__ = "0.6.1"
+__version__ = "0.6.3"
 PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"