lmnr 0.3.6__py3-none-any.whl → 0.3.7__py3-none-any.whl

lmnr/__init__.py

@@ -1,6 +1,6 @@
 from .sdk.client import Laminar
 from .sdk.decorators import observe, lmnr_context, wrap_llm_call
-from .sdk.interface import trace, TraceContext, SpanContext
+from .sdk.interface import trace, TraceContext, SpanContext, initialize
 from .sdk.tracing_types import EvaluateEvent
 from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput
 

lmnr/sdk/client.py

@@ -119,8 +119,8 @@ class Laminar:
 
     def batch_post_traces(self, batch: list[Union[Span, Trace]]):
         log = logging.getLogger("laminar.client")
-        url = self._base_url + "/v1/traces"
-        data = json.dumps({"traces": [item.to_dict() for item in batch]})
+        url = self._base_url + "/v1/observations"
+        data = json.dumps({"observations": [item.to_dict() for item in batch]})
         log.debug(f"making request to {url}")
         headers = self._headers()
         res = requests.post(url, data=data, headers=headers)

lmnr/sdk/context.py

@@ -34,6 +34,7 @@ class LaminarContextManager:
         project_api_key: str = None,
         threads: int = 1,
         max_task_queue_size: int = 1000,
+        env: dict[str, str] = {},
     ):
         self.project_api_key = project_api_key or os.environ.get("LMNR_PROJECT_API_KEY")
         if not self.project_api_key:
@@ -47,6 +48,7 @@ class LaminarContextManager:
             max_task_queue_size=max_task_queue_size,
             threads=threads,
         )
+        self.env = env
         # atexit executes functions last in first out, so we want to make sure
         # that we finalize the trace before thread manager is closed, so the updated
         # trace is sent to the server
@@ -60,7 +62,6 @@ class LaminarContextManager:
         metadata: Optional[dict[str, Any]] = None,
         attributes: Optional[dict[str, Any]] = None,
         span_type: Literal["DEFAULT", "LLM"] = "DEFAULT",
-        check_event_names: list[str] = None,
         # trace attributes
         user_id: Optional[str] = None,
         session_id: Optional[str] = None,
@@ -89,7 +90,6 @@ class LaminarContextManager:
             attributes=attributes,
             parent_span_id=parent_span_id,
             span_type=span_type,
-            check_event_names=check_event_names,
         )
         stack = _lmnr_stack_context.get()
         _lmnr_stack_context.set(stack + [span])
@@ -146,6 +146,7 @@ class LaminarContextManager:
             attributes = self._extract_llm_attributes_from_response(
                 provider=provider, response=result
             )
+
         return self._finalize_span(
             span,
             provider=provider,
@@ -159,6 +160,7 @@ class LaminarContextManager:
         metadata: Optional[dict[str, Any]] = None,
         attributes: Optional[dict[str, Any]] = None,
         evaluate_events: list[EvaluateEvent] = None,
+        events: list[Event] = None,
         override: bool = False,
     ):
         stack = _lmnr_stack_context.get()
@@ -173,6 +175,7 @@ class LaminarContextManager:
             if override
             else span.evaluateEvents + (evaluate_events or [])
         )
+        new_events = events if override else span.events + (events or [])
         new_attributes = (
             attributes
             if override
@@ -182,6 +185,7 @@ class LaminarContextManager:
             span=span,
             metadata=new_metadata,
             evaluate_events=new_evaluate_events,
+            events=new_events,
             attributes=new_attributes,
         )
 
@@ -238,7 +242,8 @@ class LaminarContextManager:
         input: Optional[Any] = None,
         metadata: Optional[dict[str, Any]] = None,
         attributes: Optional[dict[str, Any]] = None,
-        check_event_names: list[str] = None,
+        evaluate_events: Optional[list[EvaluateEvent]] = None,
+        events: Optional[list[Event]] = None,
     ) -> Span:
         """Internal method to create a span object. Use `ObservationContext.span` instead."""
         span = Span(
@@ -251,7 +256,8 @@ class LaminarContextManager:
             metadata=metadata,
             attributes=attributes,
             span_type=span_type,
-            evaluate_events=check_event_names or [],
+            evaluate_events=evaluate_events or [],
+            events=events or [],
         )
         return span
 
@@ -265,6 +271,7 @@ class LaminarContextManager:
         metadata: Optional[dict[str, Any]] = None,
         attributes: Optional[dict[str, Any]] = None,
         evaluate_events: Optional[list[EvaluateEvent]] = None,
+        events: Optional[list[Event]] = None,
         override: bool = False,
     ) -> Span:
         """Internal method to update a span object. Use `SpanContext.update()` instead."""
@@ -275,6 +282,7 @@ class LaminarContextManager:
             metadata=metadata,
             attributes=attributes,
             evaluate_events=evaluate_events,
+            events=events,
             override=override,
         )
         if finalize:
@@ -287,10 +295,14 @@ class LaminarContextManager:
         value: Optional[Union[str, int, float, bool]] = None,
         timestamp: Optional[datetime.datetime] = None,
     ):
-        span = _lmnr_stack_context.get()[-1] if _lmnr_stack_context.get() else None
-        if not span or not isinstance(span, Span):
-            self._log.warning(f"No active span to send event. Ignoring event. {name}")
+        stack = _lmnr_stack_context.get()
+        if not stack or not isinstance(stack[-1], Span):
+            self._log.warning(
+                f"No active span to add check event. Ignoring event. {name}"
+            )
             return
+
+        span = stack[-1]
         event = Event(
             name=name,
             span_id=span.id,
@@ -298,8 +310,9 @@ class LaminarContextManager:
             value=value,
         )
         span.add_event(event)
+        _lmnr_stack_context.set(stack)
 
-    def evaluate_event(self, name: str, data: str):
+    def evaluate_event(self, name: str, evaluator: str, data: dict):
         stack = _lmnr_stack_context.get()
         if not stack or not isinstance(stack[-1], Span):
             self._log.warning(
@@ -309,10 +322,13 @@ class LaminarContextManager:
         stack[-1].evaluateEvents.append(
             EvaluateEvent(
                 name=name,
+                evaluator=evaluator,
                 data=data,
                 timestamp=datetime.datetime.now(datetime.timezone.utc),
+                env=self.env,
             )
         )
+        _lmnr_stack_context.set(stack)
 
     def run_pipeline(
         self,
@@ -334,6 +350,9 @@ class LaminarContextManager:
             trace_id=trace_id,
         )
 
+    def set_env(self, env: dict[str, str]):
+        self.env = env
+
     def _force_finalize_trace(self):
         # TODO: flush in progress spans as error?
         pass

lmnr/sdk/decorators.py

@@ -155,18 +155,21 @@ class LaminarDecorator:
         laminar = LaminarSingleton().get()
         laminar.event(name, value=value, timestamp=timestamp)
 
-    def evaluate_event(self, name: str, data: str):
-        """Evaluate an event with the given name and data. The event value will be assessed by the Laminar evaluation engine.
-        Data is passed as an input to the agent, so you need to specify which data you want to evaluate. Most of the times,
-        this is an output of the LLM generation, but sometimes, you may want to evaluate the input or both. In the latter case,
-        concatenate the input and output annotating with natural language.
+    def evaluate_event(self, name: str, evaluator: str, data: dict):
+        """Evaluate an event with the given name by evaluator based on the given data.
+        Evaluator is the Laminar pipeline name.
+        Data is passed as an input to the the evaluator pipeline, so you need to specify which data you want to evaluate. The prompt
+        of the evaluator will be templated with the keys of the data dictionary.
+
+        Usually, you would want to pass the output of LLM generation, users' messages, and some other surrounding data to 'data'.
 
         Args:
-            name (str): Name of the event. Must be predefined in the Laminar events page.
-            data (str): Data to be evaluated. Typically the output of the LLM generation.
+            name (str): Name of the event.
+            evaluator (str): Name of the evaluator pipeline.
+            data (dict): Data to be used when evaluating the event.
         """
         laminar = LaminarSingleton().get()
-        laminar.evaluate_event(name, data)
+        laminar.evaluate_event(name, evaluator=evaluator, data=data)
 
     def run(
         self,

lmnr/sdk/interface.py

@@ -98,7 +98,6 @@ class SpanContext(ObservationContext):
             output (Optional[Any], optional): output of the span. Defaults to None.
             metadata (Optional[dict[str, Any]], optional): any additional metadata to the span. Defaults to None.
             attributes (Optional[dict[str, Any]], optional): pre-defined attributes (see semantic-convention). Defaults to None.
-            check_event_names (Optional[list[EvaluateEvent]], optional): List of events to evaluate for and tag. Defaults to None.
             override (bool, optional): override existing metadata fully. If False, metadata is merged. Defaults to False.
 
         Returns:
@@ -137,7 +136,6 @@ class SpanContext(ObservationContext):
             output (Optional[Any], optional): output of the span. Defaults to None.
             metadata (Optional[dict[str, Any]], optional): any additional metadata to the span. Defaults to None.
             attributes (Optional[dict[str, Any]], optional): pre-defined attributes (see semantic-convention). Defaults to None.
-            check_event_names (Optional[list[EvaluateEvent]], optional): List of events to evaluate for and tag. Defaults to None.
             override (bool, optional): override existing metadata fully. If False, metadata is merged. Defaults to False.
 
         Returns:
@@ -178,15 +176,18 @@ class SpanContext(ObservationContext):
         self.observation.add_event(event)
         return self
 
-    def evaluate_event(self, name: str, data: str) -> "SpanContext":
-        """Evaluate an event with the given name and data. The event value will be assessed by the Laminar evaluation engine.
-        Data is passed as an input to the agent, so you need to specify which data you want to evaluate. Most of the times,
-        this is an output of the LLM generation, but sometimes, you may want to evaluate the input or both. In the latter case,
-        concatenate the input and output annotating with natural language.
+    def evaluate_event(self, name: str, evaluator: str, data: dict) -> "SpanContext":
+        """Evaluate an event with the given name by evaluator based on the given data.
+        Evaluator is the Laminar pipeline name.
+        Data is passed as an input to the the evaluator pipeline, so you need to specify which data you want to evaluate. The prompt
+        of the evaluator will be templated with the keys of the data dictionary.
+
+        Usually, you would want to pass the output of LLM generation, users' messages, and some other surrounding data to 'data'.
 
         Args:
-            name (str): Name of the event. Must be predefined in the Laminar events page.
-            data (str): Data to be evaluated. Typically the output of the LLM generation.
+            name (str): Name of the event.
+            evaluator (str): Name of the evaluator pipeline.
+            data (dict): Data to be used when evaluating the event.
 
         Returns:
             SpanContext: the updated span context
@@ -197,6 +198,7 @@ class SpanContext(ObservationContext):
             evaluate_events=[
                 EvaluateEvent(
                     name=name,
+                    evaluator=evaluator,
                     data=data,
                     timestamp=datetime.datetime.now(datetime.timezone.utc),
                 )
@@ -211,6 +213,7 @@ class SpanContext(ObservationContext):
         metadata: Optional[dict[str, Any]] = None,
         attributes: Optional[dict[str, Any]] = None,
         evaluate_events: Optional[list[EvaluateEvent]] = None,
+        events: Optional[list[Event]] = None,
         override: bool = False,
         finalize: bool = False,
     ) -> "SpanContext":
@@ -306,3 +309,8 @@ def trace(
         release=release,
     )
     return TraceContext(trace, None)
+
+
+def initialize(env: dict[str, str]) -> None:
+    laminar = LaminarSingleton().get()
+    laminar.set_env(env)

lmnr/sdk/tracing_types.py

@@ -8,9 +8,20 @@ from .utils import to_dict
 
 
 class EvaluateEvent(pydantic.BaseModel):
+    """
+    EvaluateEvent is an event which need to be evaluated on the server.
+
+    Args:
+        env: dict[str, str]: Environment variables to be used during evaluation.
+            It is optional and can be left empty, because it will be merged with LaminarContextManager's env.
+            So you need to only set it once there.
+    """
+
     name: str
-    data: str
+    evaluator: str
+    data: dict
     timestamp: Optional[datetime.datetime] = None
+    env: dict[str, str] = {}
 
 
 class Span(pydantic.BaseModel):
@@ -29,7 +40,7 @@ class Span(pydantic.BaseModel):
     output: Optional[Any] = None
     metadata: Optional[dict[str, Any]] = None
     evaluateEvents: list[EvaluateEvent] = []
-    events: list["Event"] = None
+    events: list["Event"] = []
 
     def __init__(
         self,
@@ -44,6 +55,7 @@ class Span(pydantic.BaseModel):
         metadata: Optional[dict[str, Any]] = {},
         attributes: Optional[dict[str, Any]] = {},
         evaluate_events: list[EvaluateEvent] = [],
+        events: list["Event"] = [],
     ):
         super().__init__(
             version=version,
@@ -57,7 +69,7 @@ class Span(pydantic.BaseModel):
             metadata=metadata or {},
             attributes=attributes or {},
             evaluateEvents=evaluate_events,
-            events=[],
+            events=events,
         )
 
     def update(
@@ -68,6 +80,7 @@ class Span(pydantic.BaseModel):
         metadata: Optional[dict[str, Any]] = None,
         attributes: Optional[dict[str, Any]] = None,
         evaluate_events: Optional[list[EvaluateEvent]] = None,
+        events: Optional[list["Event"]] = None,
         override: bool = False,
     ):
         self.endTime = end_time or datetime.datetime.now(datetime.timezone.utc)
@@ -86,9 +99,11 @@ class Span(pydantic.BaseModel):
             if override
             else self.evaluateEvents + (evaluate_events or [])
         )
+        new_events = events or [] if override else self.events + (events or [])
         self.metadata = new_metadata
         self.attributes = new_attributes
         self.evaluateEvents = new_evaluate_events
+        self.events = new_events
 
     def add_event(self, event: "Event"):
         self.events.append(event)
@@ -106,6 +121,7 @@ class Span(pydantic.BaseModel):
                     if isinstance(value, pydantic.BaseModel)
                     else value
                 )
+
         obj = to_dict(obj)
         return obj
 

{lmnr-0.3.6.dist-info → lmnr-0.3.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.3.6
+Version: 0.3.7
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -38,15 +38,9 @@ Read more [here](https://docs.lmnr.ai/api-reference/introduction#authentication)
 
 You can send events in two ways:
 - `.event(name, value)` – for a pre-defined event with one of possible values.
-- `.evaluate_event(name, data)` – for an event that our agent checks for and assigns a value from possible values.
+- `.evaluate_event(name, evaluator, data)` – for an event that is evaluated by evaluator pipeline based on the data.
 
-There are 3 types of events:
-- Number - Numeric value.
-- String - Arbitrary string.
-- Boolean - Convenient to classify if something has took place or not.
-
-Important notes:
-- If event name does not match anything pre-defined in the UI, the event won't be saved.
+Read our [docs](https://docs.lmnr.ai) to learn more about event types, how they are created and evaluated, etc.
 
 ## Instrumentation
 
@@ -70,9 +64,17 @@ You can also import `lmnr_context` in order to interact and have more control ov
 import os
 from openai import OpenAI
 
-from lmnr import observe, wrap_llm_call, lmnr_context
+from lmnr import observe, wrap_llm_call, lmnr_context, initialize
 client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
+# add if your online evaluation pipelines need these keys
+initialize(
+    {
+        "OPENAI_API_KEY": "sk-...",
+        "ANTHROPIC_API_KEY": "sk-...",
+    }
+)
+
 @observe()  # annotate all functions you want to trace
 def poem_writer(topic="turbulence"):
     prompt = f"write a poem about {topic}"
@@ -93,7 +95,7 @@ def poem_writer(topic="turbulence"):
         lmnr_context.event("topic_alignment", "good")
     
     # to trigger an automatic check for a possible event do:
-    lmnr_context.evaluate_event("excessive_wordiness", poem)
+    lmnr_context.evaluate_event("excessive_wordiness", "wordiness_evaluator", {"poem": poem})
 
     return poem
 
@@ -119,7 +121,7 @@ Both `TraceContext` and `SpanContext` expose the following interfaces:
 
 In addition, `SpanContext` allows you to:
 - `event(name: str, value: str | int)` - emit a custom event at any point
-- `evaluate_event(name: str, data: str)` - register a possible event for automatic checking by Laminar.
+- `evaluate_event(name: str, evaluator: str, data: dict)` - register a possible event for automatic checking by Laminar's evaluator pipeline.
 - `end(**kwargs)` – update the current span, and terminate it
 
 Example:
@@ -128,10 +130,18 @@ Example:
 import os
 from openai import OpenAI
 
-from lmnr import trace, TraceContext, SpanContext, EvaluateEvent
+from lmnr import trace, TraceContext, SpanContext, EvaluateEvent, initialize
 from lmnr.semantic_conventions.gen_ai_spans import INPUT_TOKEN_COUNT, OUTPUT_TOKEN_COUNT, RESPONSE_MODEL, PROVIDER, STREAM
 client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
+# add if your online evaluation pipelines need these keys
+initialize(
+    {
+        "OPENAI_API_KEY": "sk-...",
+        "ANTHROPIC_API_KEY": "sk-...",
+    }
+)
+
 def poem_writer(t: TraceContext, topic = "turbulence"):
     span: SpanContext = t.span(name="poem_writer", input=topic)
 
@@ -154,11 +164,10 @@ def poem_writer(t: TraceContext, topic = "turbulence"):
     if topic in poem:
         llm_span.event("topic_alignment", "good")  # send an event with a pre-defined name
 
-    # note that you can register possible events here as well,
-    # not only `llm_span.evaluate_event()`
+    llm_span.evaluate_event("positiveness", "positiveness_evaluator", {"poem": poem})
+
     llm_span.end(
         output=poem,
-        evaluate_events=[EvaluateEvent(name="excessive_wordines", data=poem)],
         attributes={
             INPUT_TOKEN_COUNT: response.usage.prompt_tokens,
             OUTPUT_TOKEN_COUNT: response.usage.completion_tokens,

{lmnr-0.3.6.dist-info → lmnr-0.3.7.dist-info}/RECORD

@@ -1,23 +1,23 @@
-lmnr/__init__.py,sha256=vUiBEqNVi-dWlFKqnyxt0387t1kzVXrwFp7ShhDJyXY,324
+lmnr/__init__.py,sha256=hHpH10FQOUuTfwPDJW35ZWSYjLKNaLhm-xpW2-iIWdk,336
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lmnr/sdk/client.py,sha256=e6cIvJq38a6XAU8FGWYtNXVAPlEoZEyKS7hC3M_6EkU,5749
+lmnr/sdk/client.py,sha256=sFWORPHh0iPOOIMy35VpE7YaDW-oFcTKiP0wQi-Ay1I,5761
 lmnr/sdk/collector.py,sha256=6LRmPhOcmGplUDWm_sJh0dVrLTHknd_kmq7POGuAvoQ,5338
 lmnr/sdk/constants.py,sha256=USCfwuUqRx6_0xC8WUxqGj766dInqQkWJcf8U5vPK7s,34
-lmnr/sdk/context.py,sha256=4ngtUYhix91hUW4Kpo-WplDiNs-SCqz6LmHMl84StoA,15398
-lmnr/sdk/decorators.py,sha256=vEGXYJUKGNVAVEt3sRKAUTtGAdhfQbyA474kfHsZHTk,12024
-lmnr/sdk/interface.py,sha256=ugtNIcHZhHC_qSyu_-1Fm62_phOb4vW-Lo5VZ1WaMhc,12255
+lmnr/sdk/context.py,sha256=erEdKFElK_I1aWNUbqlb7rJ7A_zHPX7aks52P9WXWZw,15926
+lmnr/sdk/decorators.py,sha256=ACaspfBDVjHeBPp0-YNfyCy99SDZrF6klSG80Towrqg,12086
+lmnr/sdk/interface.py,sha256=W5yq-Y9EsCoTBIWWwxV7w542wtIZ2b3f9VP6Bh7HDUw,12227
 lmnr/sdk/providers/__init__.py,sha256=wNCgQnt9-bnTNXLQWdPgyKhqA1ajiaEd1Rr2KPOpazM,54
 lmnr/sdk/providers/base.py,sha256=xc6iA8yY_VK6tbzswt-3naZ53aAXtOLur9j8eimC_ZA,1054
 lmnr/sdk/providers/fallback.py,sha256=9-srLJgDK5CWD8DIdnxo6jbSsnwDvdHC-vA06BfEkqQ,5431
 lmnr/sdk/providers/openai.py,sha256=9X2VWz4_EqQBEA1UEww2EKdqods6IzOEmaO6dnhY-Lw,5725
 lmnr/sdk/providers/utils.py,sha256=ROt82VrvezExYOxionAynD3dp6oX5JoPW6F1ayTm7q8,946
-lmnr/sdk/tracing_types.py,sha256=O6sfTBRKxz8WtRy4uEr-_wkSqu58mmEa1SXXJXDGwBI,6267
+lmnr/sdk/tracing_types.py,sha256=hbxRSkC2PHjgqaL6RRqp1Ev1C5U5-o_sNZH5QqtDHAw,6858
 lmnr/sdk/types.py,sha256=5-Ft-l35wtmn2xxE8BTqsM3nx1zD799tRv4qiOkED50,2121
 lmnr/sdk/utils.py,sha256=9ScSKalwZbW5wAhGN_Mn26_LZewVBkTz_OG5uQi6Og4,3401
 lmnr/semantic_conventions/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/semantic_conventions/gen_ai_spans.py,sha256=3s-2J5v3t5LcMKwK2DefPn56XpxN5oMEYtb9Mf9D_gA,1541
-lmnr-0.3.6.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.3.6.dist-info/METADATA,sha256=OcWavyUecfR_YwlTJF003WghB4mDqbvUg0ril4arC0k,9485
-lmnr-0.3.6.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lmnr-0.3.6.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
-lmnr-0.3.6.dist-info/RECORD,,
+lmnr-0.3.7.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.3.7.dist-info/METADATA,sha256=5QEiLgfp5HJ0EI67nKkQt_bcqkWyKjz8d-_Zb6rUc2g,9654
+lmnr-0.3.7.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lmnr-0.3.7.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
+lmnr-0.3.7.dist-info/RECORD,,