lmnr 0.6.3__py3-none-any.whl → 0.6.4__py3-none-any.whl

lmnr/sdk/evaluations.py

@@ -3,7 +3,7 @@ import re
 import uuid
 
 from tqdm import tqdm
-from typing import Any, Awaitable, Optional, Set, Union
+from typing import Any, Awaitable
 
 from lmnr.opentelemetry_lib.tracing.instruments import Instruments
 from lmnr.opentelemetry_lib.tracing.attributes import SPAN_TYPE
@@ -33,7 +33,7 @@ MAX_EXPORT_BATCH_SIZE = 64
 
 
 def get_evaluation_url(
-    project_id: str, evaluation_id: str, base_url: Optional[str] = None
+    project_id: str, evaluation_id: str, base_url: str | None = None
 ):
     if not base_url or base_url == "https://api.lmnr.ai":
         base_url = "https://www.lmnr.ai"
@@ -95,32 +95,34 @@ class EvaluationReporter:
 class Evaluation:
     def __init__(
         self,
-        data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
+        data: EvaluationDataset | list[Datapoint | dict],
         executor: Any,
         evaluators: dict[str, EvaluatorFunction],
         human_evaluators: list[HumanEvaluator] = [],
-        name: Optional[str] = None,
-        group_name: Optional[str] = None,
+        name: str | None = None,
+        group_name: str | None = None,
         concurrency_limit: int = DEFAULT_BATCH_SIZE,
-        project_api_key: Optional[str] = None,
-        base_url: Optional[str] = None,
-        http_port: Optional[int] = None,
-        grpc_port: Optional[int] = None,
-        instruments: Optional[Set[Instruments]] = None,
-        max_export_batch_size: Optional[int] = MAX_EXPORT_BATCH_SIZE,
-        trace_export_timeout_seconds: Optional[int] = None,
+        project_api_key: str | None = None,
+        base_url: str | None = None,
+        http_port: int | None = None,
+        grpc_port: int | None = None,
+        instruments: set[Instruments] | None = None,
+        max_export_batch_size: int | None = MAX_EXPORT_BATCH_SIZE,
+        trace_export_timeout_seconds: int | None = None,
     ):
         """
         Initializes an instance of the Evaluation class.
 
         Parameters:
-            data (Union[List[EvaluationDatapoint|dict], EvaluationDataset]):\
+            data (list[Datapoint|dict] | EvaluationDataset):\
                 List of data points to evaluate or an evaluation dataset.
-                            `data` is the input to the executor function,
-                            `target` is the input to the evaluator function.
+                    `data` is the input to the executor function.
+                    `target` is the input to the evaluator function.
+                    `metadata` is optional metadata to associate with the\
+                        datapoint.
             executor (Callable[..., Any]): The executor function.\
-                            Takes the data point + any additional arguments\
-                            and returns the output to evaluate.
+                    Takes the data point + any additional arguments and returns\
+                    the output to evaluate.
             evaluators (dict[str, Callable[..., Any]]): Evaluator functions and\
                 names. Each evaluator function takes the output of the executor\
                 _and_ the target data, and returns a score. The score can be a\
@@ -132,30 +134,31 @@ class Evaluation:
                 [Beta] List of instances of HumanEvaluator. For now, human\
                 evaluator only holds the queue name.
                 Defaults to an empty list.
-            name (Optional[str], optional): Optional name of the evaluation.\
+            name (str | None, optional): Optional name of the evaluation.\
                 Used to identify the evaluation in the group.\
                 If not provided, a random name will be generated.
                 Defaults to None.
-            group_name (Optional[str], optional): an identifier to group\
+            group_name (str | None, optional): an identifier to group\
                 evaluations. Only evaluations within the same group_name can be\
                 visually compared. If not provided, "default" is assigned.
                 Defaults to None
-            concurrency_limit (int, optional): The concurrency limit for evaluation. This many\
-                data points will be evaluated in parallel with a pool of workers.
+            concurrency_limit (int, optional): The concurrency limit for\
+                evaluation. This many data points will be evaluated in parallel\
+                with a pool of workers.
                 Defaults to DEFAULT_BATCH_SIZE.
-            project_api_key (Optional[str], optional): The project API key.\
+            project_api_key (str | None, optional): The project API key.\
                 If not provided, LMNR_PROJECT_API_KEY environment variable is\
                 used.
                 Defaults to an empty string.
-            base_url (Optional[str], optional): The base URL for Laminar API.\
+            base_url (str | None, optional): The base URL for Laminar API.\
                 Useful if self-hosted. Do NOT include the port, use `http_port`\
                 and `grpc_port` instead.
                 Defaults to "https://api.lmnr.ai".
-            http_port (Optional[int], optional): The port for Laminar API\
+            http_port (int | None, optional): The port for Laminar API\
                 HTTP service. Defaults to 443 if not specified.
-            grpc_port (Optional[int], optional): The port for Laminar API\
+            grpc_port (int | None, optional): The port for Laminar API\
                 gRPC service. Defaults to 8443 if not specified.
-            instruments (Optional[Set[Instruments]], optional): Set of modules\
+            instruments (set[Instruments] | None, optional): Set of modules\
                 to auto-instrument. If None, all available instruments will be\
                 used.
                 See https://docs.lmnr.ai/tracing/automatic-instrumentation
@@ -393,21 +396,21 @@ class Evaluation:
 
 
 def evaluate(
-    data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
+    data: EvaluationDataset | list[Datapoint | dict],
     executor: ExecutorFunction,
     evaluators: dict[str, EvaluatorFunction],
     human_evaluators: list[HumanEvaluator] = [],
-    name: Optional[str] = None,
-    group_name: Optional[str] = None,
+    name: str | None = None,
+    group_name: str | None = None,
     concurrency_limit: int = DEFAULT_BATCH_SIZE,
-    project_api_key: Optional[str] = None,
-    base_url: Optional[str] = None,
-    http_port: Optional[int] = None,
-    grpc_port: Optional[int] = None,
-    instruments: Optional[Set[Instruments]] = None,
-    max_export_batch_size: Optional[int] = MAX_EXPORT_BATCH_SIZE,
-    trace_export_timeout_seconds: Optional[int] = None,
-) -> Optional[Awaitable[None]]:
+    project_api_key: str | None = None,
+    base_url: str | None = None,
+    http_port: int | None = None,
+    grpc_port: int | None = None,
+    instruments: set[Instruments] | None = None,
+    max_export_batch_size: int | None = MAX_EXPORT_BATCH_SIZE,
+    trace_export_timeout_seconds: int | None = None,
+) -> Awaitable[None] | None:
     """
     If added to the file which is called through `lmnr eval` command, then
     registers the evaluation; otherwise, runs the evaluation.
@@ -418,7 +421,7 @@ def evaluate(
     You must await the call to `evaluate`.
 
     Parameters:
-        data (Union[list[EvaluationDatapoint|dict]], EvaluationDataset]):\
+        data (list[EvaluationDatapoint|dict] | EvaluationDataset):\
             List of data points to evaluate or an evaluation dataset.
                 `data` is the input to the executor function,
                 `target` is the input to the evaluator function.
@@ -437,33 +440,33 @@ def evaluate(
             [Beta] List of instances of HumanEvaluator. For now, human\
             evaluator only holds the queue name.
             Defaults to an empty list.
-        name (Optional[str], optional): Optional name of the evaluation.\
+        name (str | None, optional): Optional name of the evaluation.\
             Used to identify the evaluation in the group. If not provided, a\
             random name will be generated.
             Defaults to None.
-        group_name (Optional[str], optional): An identifier to group evaluations.\
+        group_name (str | None, optional): An identifier to group evaluations.\
             Only evaluations within the same group_name can be visually compared.\
             If not provided, set to "default".
             Defaults to None
         concurrency_limit (int, optional): The concurrency limit for evaluation.
                         Defaults to DEFAULT_BATCH_SIZE.
-        project_api_key (Optional[str], optional): The project API key.
+        project_api_key (str | None, optional): The project API key.
                         Defaults to None.
-        base_url (Optional[str], optional): The base URL for Laminar API.\
+        base_url (str | None, optional): The base URL for Laminar API.\
                         Useful if self-hosted elsewhere. Do NOT include the\
                         port, use `http_port` and `grpc_port` instead.
                         Defaults to "https://api.lmnr.ai".
-        http_port (Optional[int], optional): The port for Laminar API's HTTP\
+        http_port (int | None, optional): The port for Laminar API's HTTP\
                         service. 443 is used if not specified.
                         Defaults to None.
-        grpc_port (Optional[int], optional): The port for Laminar API's gRPC\
+        grpc_port (int | None, optional): The port for Laminar API's gRPC\
                         service. 8443 is used if not specified.
                         Defaults to None.
-        instruments (Optional[Set[Instruments]], optional): Set of modules to\
+        instruments (set[Instruments] | None, optional): Set of modules to\
                         auto-instrument. If None, all available instruments\
                         will be used.
                         Defaults to None.
-        trace_export_timeout_seconds (Optional[int], optional): The timeout for\
+        trace_export_timeout_seconds (int | None, optional): The timeout for\
                         trace export on OpenTelemetry exporter. Defaults to None.
     """
     evaluation = Evaluation(

lmnr/sdk/laminar.py

@@ -12,10 +12,11 @@ from lmnr.opentelemetry_lib import MAX_MANUAL_SPAN_PAYLOAD_SIZE
 from lmnr.opentelemetry_lib.decorators import json_dumps
 from opentelemetry import context as context_api, trace
 from opentelemetry.context import attach, detach
+from opentelemetry.trace import INVALID_TRACE_ID
 from opentelemetry.sdk.trace.id_generator import RandomIdGenerator
 from opentelemetry.util.types import AttributeValue
 
-from typing import Any, Literal, Optional, Set, Union
+from typing import Any, Literal
 
 import copy
 import datetime
@@ -48,22 +49,22 @@ from .types import (
 
 
 class Laminar:
-    __project_api_key: Optional[str] = None
+    __project_api_key: str | None = None
     __initialized: bool = False
-    __base_http_url: Optional[str] = None
+    __base_http_url: str | None = None
 
     @classmethod
     def initialize(
         cls,
-        project_api_key: Optional[str] = None,
-        base_url: Optional[str] = None,
-        http_port: Optional[int] = None,
-        grpc_port: Optional[int] = None,
-        instruments: Optional[Set[Instruments]] = None,
-        disabled_instruments: Optional[Set[Instruments]] = None,
+        project_api_key: str | None = None,
+        base_url: str | None = None,
+        http_port: int | None = None,
+        grpc_port: int | None = None,
+        instruments: set[Instruments] | None = None,
+        disabled_instruments: set[Instruments] | None = None,
         disable_batch: bool = False,
-        max_export_batch_size: Optional[int] = None,
-        export_timeout_seconds: Optional[int] = None,
+        max_export_batch_size: int | None = None,
+        export_timeout_seconds: int | None = None,
         set_global_tracer_provider: bool = True,
         otel_logger_level: int = logging.ERROR,
     ):
@@ -72,31 +73,31 @@ class Laminar:
         decorators.
 
         Args:
-            project_api_key (Optional[str], optional): Laminar project api key.\
+            project_api_key (str | None, optional): Laminar project api key.\
                             You can generate one by going to the projects\
                             settings page on the Laminar dashboard.\
                             If not specified, it will try to read from the\
                             LMNR_PROJECT_API_KEY environment variable\
                             in os.environ or in .env file.
                             Defaults to None.
-            base_url (Optional[str], optional): Laminar API url. Do NOT include\
+            base_url (str | None, optional): Laminar API url. Do NOT include\
                             the port number, use `http_port` and `grpc_port`.\
                             If not specified, defaults to https://api.lmnr.ai.
-            http_port (Optional[int], optional): Laminar API http port.\
+            http_port (int | None, optional): Laminar API http port.\
                             If not specified, defaults to 443.
-            grpc_port (Optional[int], optional): Laminar API grpc port.\
+            grpc_port (int | None, optional): Laminar API grpc port.\
                             If not specified, defaults to 8443.
-            instruments (Optional[Set[Instruments]], optional): Instruments to\
+            instruments (set[Instruments] | None, optional): Instruments to\
                         enable. Defaults to all instruments. You can pass\
                         an empty set to disable all instruments. Read more:\
                         https://docs.lmnr.ai/tracing/automatic-instrumentation
-            disabled_instruments (Optional[Set[Instruments]], optional): Instruments to\
+            disabled_instruments (set[Instruments] | None, optional): Instruments to\
                         disable. Defaults to None.
             disable_batch (bool, optional): If set to True, spans will be sent\
                         immediately to the backend. Useful for debugging, but\
                         may cause performance overhead in production.
                         Defaults to False.
-            export_timeout_seconds (Optional[int], optional): Timeout for the OTLP\
+            export_timeout_seconds (int | None, optional): Timeout for the OTLP\
                         exporter. Defaults to 30 seconds (unlike the\
                         OpenTelemetry default of 10 seconds).
                         Defaults to None.
@@ -176,8 +177,8 @@ class Laminar:
     def event(
         cls,
         name: str,
-        value: Optional[AttributeValue] = None,
-        timestamp: Optional[Union[datetime.datetime, int]] = None,
+        value: AttributeValue | None = None,
+        timestamp: datetime.datetime | int | None = None,
     ):
         """Associate an event with the current span. If using manual\
         instrumentation, use raw OpenTelemetry `span.add_event()` instead.\
@@ -185,14 +186,13 @@ class Laminar:
 
         Args:
             name (str): event name
-            value (Optional[AttributeValue]): event value. Must be a primitive\
-                            type. Boolean true is assumed in the backend if\
-                            `value` is None.
-                            Defaults to None.
-            timestamp (Optional[Union[datetime.datetime, int]], optional):\
-                            If int, must be epoch nanoseconds. If not\
-                            specified, relies on the underlying OpenTelemetry\
-                            implementation. Defaults to None.
+            value (AttributeValue | None, optional): event value. Must be a\
+                primitive type. Boolean `True` is assumed in the backend if\
+                `value` is None.
+                Defaults to None.
+            timestamp (datetime.datetime | int | None, optional): If int, must\
+                be epoch nanoseconds. If not specified, relies on the underlying\
+                OpenTelemetry implementation. Defaults to None.
         """
         if timestamp and isinstance(timestamp, datetime.datetime):
             timestamp = int(timestamp.timestamp() * 1e9)
@@ -220,12 +220,10 @@ class Laminar:
         cls,
         name: str,
         input: Any = None,
-        span_type: Union[
-            Literal["DEFAULT"], Literal["LLM"], Literal["TOOL"]
-        ] = "DEFAULT",
-        context: Optional[Context] = None,
-        labels: Optional[list[str]] = None,
-        parent_span_context: Optional[LaminarSpanContext] = None,
+        span_type: Literal["DEFAULT", "LLM", "TOOL"] = "DEFAULT",
+        context: Context | None = None,
+        labels: list[str] | None = None,
+        parent_span_context: LaminarSpanContext | None = None,
     ):
         """Start a new span as the current span. Useful for manual
         instrumentation. If `span_type` is set to `"LLM"`, you should report
@@ -243,12 +241,12 @@ class Laminar:
             name (str): name of the span
             input (Any, optional): input to the span. Will be sent as an\
                 attribute, so must be json serializable. Defaults to None.
-            span_type (Union[Literal["DEFAULT"], Literal["LLM"]], optional):\
+            span_type (Literal["DEFAULT", "LLM", "TOOL"], optional):\
                 type of the span. If you use `"LLM"`, you should report usage\
                 and response attributes manually. Defaults to "DEFAULT".
-            context (Optional[Context], optional): raw OpenTelemetry context\
+            context (Context | None, optional): raw OpenTelemetry context\
                 to attach the span to. Defaults to None.
-            parent_span_context (Optional[LaminarSpanContext], optional): parent\
+            parent_span_context (LaminarSpanContext | None, optional): parent\
                 span context to use for the span. Useful for continuing traces\
                 across services. If parent_span_context is a\
                 raw OpenTelemetry span context, or if it is a dictionary or string\
@@ -258,7 +256,7 @@ class Laminar:
                 `Laminar.get_span_context`, `Laminar.get_span_context_dict` and\
                 `Laminar.get_span_context_str` for more information.
                 Defaults to None.
-            labels (Optional[list[str]], optional): labels to set for the\
+            labels (list[str] | None, optional): labels to set for the\
                 span. Defaults to None.
         """
 
@@ -321,7 +319,7 @@ class Laminar:
 
     @classmethod
     @contextmanager
-    def with_labels(cls, labels: list[str], context: Optional[Context] = None):
+    def with_labels(cls, labels: list[str], context: Context | None = None):
         """Set labels for spans within this `with` context. This is useful for
         adding labels to the spans created in the auto-instrumentations.
 
@@ -363,12 +361,10 @@ class Laminar:
         cls,
         name: str,
         input: Any = None,
-        span_type: Union[
-            Literal["DEFAULT"], Literal["LLM"], Literal["TOOL"]
-        ] = "DEFAULT",
-        context: Optional[Context] = None,
-        parent_span_context: Optional[LaminarSpanContext] = None,
-        labels: Optional[dict[str, str]] = None,
+        span_type: Literal["DEFAULT", "LLM", "TOOL"] = "DEFAULT",
+        context: Context | None = None,
+        parent_span_context: LaminarSpanContext | None = None,
+        labels: dict[str, str] | None = None,
     ):
         """Start a new span. Useful for manual instrumentation.
         If `span_type` is set to `"LLM"`, you should report usage and response
@@ -405,12 +401,12 @@ class Laminar:
             name (str): name of the span
             input (Any, optional): input to the span. Will be sent as an\
                 attribute, so must be json serializable. Defaults to None.
-            span_type (Union[Literal["DEFAULT"], Literal["LLM"]], optional):\
+            span_type (Literal["DEFAULT", "LLM", "TOOL"], optional):\
                 type of the span. If you use `"LLM"`, you should report usage\
                 and response attributes manually. Defaults to "DEFAULT".
-            context (Optional[Context], optional): raw OpenTelemetry context\
+            context (Context | None, optional): raw OpenTelemetry context\
                 to attach the span to. Defaults to None.
-            parent_span_context (Optional[LaminarSpanContext], optional): parent\
+            parent_span_context (LaminarSpanContext | None, optional): parent\
                 span context to use for the span. Useful for continuing traces\
                 across services. If parent_span_context is a\
                 raw OpenTelemetry span context, or if it is a dictionary or string\
@@ -420,7 +416,7 @@ class Laminar:
                 `Laminar.get_span_context`, `Laminar.get_span_context_dict` and\
                 `Laminar.get_span_context_str` for more information.
                 Defaults to None.
-            labels (Optional[dict[str, str]], optional): labels to set for the\
+            labels (dict[str, str] | None, optional): labels to set for the\
                 span. Defaults to None.
         """
         if not cls.is_initialized():
@@ -574,8 +570,8 @@ class Laminar:
 
     @classmethod
     def get_laminar_span_context(
-        cls, span: Optional[trace.Span] = None
-    ) -> Optional[LaminarSpanContext]:
+        cls, span: trace.Span | None = None
+    ) -> LaminarSpanContext | None:
         """Get the laminar span context for a given span.
         If no span is provided, the current active span will be used.
         """
@@ -590,15 +586,15 @@ class Laminar:
 
     @classmethod
     def get_laminar_span_context_dict(
-        cls, span: Optional[trace.Span] = None
-    ) -> Optional[dict]:
+        cls, span: trace.Span | None = None
+    ) -> dict | None:
         span_context = cls.get_laminar_span_context(span)
         if span_context is None:
             return None
         return span_context.model_dump()
 
     @classmethod
-    def serialize_span_context(cls, span: Optional[trace.Span] = None) -> Optional[str]:
+    def serialize_span_context(cls, span: trace.Span | None = None) -> str | None:
         """Get the laminar span context for a given span as a string.
         If no span is provided, the current active span will be used.
 
@@ -632,9 +628,7 @@ class Laminar:
         return str(span_context)
 
     @classmethod
-    def deserialize_span_context(
-        cls, span_context: Union[dict, str]
-    ) -> LaminarSpanContext:
+    def deserialize_span_context(cls, span_context: dict | str) -> LaminarSpanContext:
         return LaminarSpanContext.deserialize(span_context)
 
     @classmethod
@@ -657,14 +651,14 @@ class Laminar:
     @classmethod
     def set_session(
         cls,
-        session_id: Optional[str] = None,
+        session_id: str | None = None,
     ):
         """Set the session and user id for the current span and the context
         (i.e. any children spans created from the current span in the current
         thread).
 
         Args:
-            session_id (Optional[str], optional): Custom session id.\
+            session_id (str | None, optional): Custom session id.\
                             Useful to debug and group long-running\
                             sessions/conversations.
                             Defaults to None.
@@ -710,6 +704,20 @@ class Laminar:
     def get_project_api_key(cls):
         return cls.__project_api_key
 
+    @classmethod
+    def get_trace_id(cls) -> uuid.UUID | None:
+        """Get the trace id for the current active span represented as a UUID.
+        Returns None if there is no active span.
+
+        Returns:
+            uuid.UUID | None: The trace id for the current span, or None if\
+            there is no active span.
+        """
+        trace_id = trace.get_current_span().get_span_context().trace_id
+        if trace_id == INVALID_TRACE_ID:
+            return None
+        return uuid.UUID(int=trace_id)
+
     @classmethod
     def _headers(cls):
         assert cls.__project_api_key is not None, "Project API key is not set"