lmnr 0.4.10__py3-none-any.whl → 0.4.12__py3-none-any.whl

lmnr/sdk/laminar.py

@@ -5,7 +5,6 @@ from opentelemetry.trace import (
     get_current_span,
     SpanKind,
 )
-from opentelemetry.semconv_ai import SpanAttributes
 from opentelemetry.util.types import AttributeValue
 from opentelemetry.context.context import Context
 from opentelemetry.util import types
@@ -26,7 +25,17 @@ import os
 import requests
 import uuid
 
-from lmnr.traceloop_sdk.tracing.tracing import set_association_properties, update_association_properties
+from lmnr.traceloop_sdk.tracing.attributes import (
+    SESSION_ID,
+    SPAN_INPUT,
+    SPAN_OUTPUT,
+    TRACE_TYPE,
+    USER_ID,
+)
+from lmnr.traceloop_sdk.tracing.tracing import (
+    set_association_properties,
+    update_association_properties,
+)
 
 from .log import VerboseColorfulFormatter
 
@@ -37,11 +46,14 @@ from .types import (
     PipelineRunResponse,
     NodeInput,
     PipelineRunRequest,
+    TraceType,
+    UpdateEvaluationResponse,
 )
 
 
 class Laminar:
-    __base_url: str = "https://api.lmnr.ai:8443"
+    __base_http_url: str
+    __base_grpc_url: str
     __project_api_key: Optional[str] = None
     __env: dict[str, str] = {}
     __initialized: bool = False
@@ -52,6 +64,8 @@ class Laminar:
         project_api_key: Optional[str] = None,
         env: dict[str, str] = {},
         base_url: Optional[str] = None,
+        http_port: Optional[int] = None,
+        grpc_port: Optional[int] = None,
         instruments: Optional[Set[Instruments]] = None,
     ):
         """Initialize Laminar context across the application.
@@ -71,13 +85,12 @@ class Laminar:
                             overriden at request time. Usually, model
                             provider keys are stored here.
                             Defaults to {}.
-            base_url (Optional[str], optional): Url of Laminar endpoint,
-                            or the  customopen telemetry ingester.
-                            If not specified, defaults to
-                            https://api.lmnr.ai:8443.
-                            For locally hosted Laminar, default setting
-                            must be http://localhost:8001
-                            Defaults to None.
+            base_url (Optional[str], optional): Laminar API url.
+                            If not specified, defaults to https://api.lmnr.ai.
+            http_port (Optional[int], optional): Laminar API http port.
+                            If not specified, defaults to 443.
+            grpc_port (Optional[int], optional): Laminar API grpc port.
+                            If not specified, defaults to 8443.
 
         Raises:
             ValueError: If project API key is not set
@@ -85,7 +98,7 @@ class Laminar:
         cls.__project_api_key = project_api_key or os.environ.get(
             "LMNR_PROJECT_API_KEY"
         )
-        if not project_api_key:
+        if not cls.__project_api_key:
             dotenv_path = dotenv.find_dotenv(usecwd=True)
             cls.__project_api_key = dotenv.get_key(
                 dotenv_path=dotenv_path, key_to_get="LMNR_PROJECT_API_KEY"
@@ -96,14 +109,16 @@ class Laminar:
                 " your project API key or set the LMNR_PROJECT_API_KEY"
                 " environment variable in your environment or .env file"
             )
-        if base_url is not None:
-            cls.__base_url = base_url
+
+        cls.__base_http_url = f"{base_url or 'https://api.lmnr.ai'}:{http_port or 443}"
+        cls.__base_grpc_url = f"{base_url or 'https://api.lmnr.ai'}:{grpc_port or 8443}"
+
         cls.__env = env
         cls.__initialized = True
         cls._initialize_logger()
         Traceloop.init(
             exporter=OTLPSpanExporter(
-                endpoint=cls.__base_url,
+                endpoint=cls.__base_grpc_url,
                 headers={"authorization": f"Bearer {cls.__project_api_key}"},
             ),
             instruments=instruments,
@@ -190,7 +205,7 @@ class Laminar:
             raise ValueError(f"Invalid request: {e}")
 
         response = requests.post(
-            cls.__base_url + "/v1/pipeline/run",
+            cls.__base_http_url + "/v1/pipeline/run",
             data=json.dumps(request.to_dict()),
             headers=cls._headers(),
         )
@@ -292,7 +307,7 @@ class Laminar:
             )
             return
 
-        current_span.add_event(name, event)
+        current_span.add_event(name, event, timestamp)
 
     @classmethod
     @contextmanager
@@ -351,8 +366,8 @@ class Laminar:
             ) as span:
                 if input is not None:
                     span.set_attribute(
-                        SpanAttributes.TRACELOOP_ENTITY_INPUT,
-                        json.dumps({"input": input}),
+                        SPAN_INPUT,
+                        json.dumps(input),
                     )
                 yield span
 
@@ -366,9 +381,7 @@ class Laminar:
         """
         span = get_current_span()
         if output is not None and span != INVALID_SPAN:
-            span.set_attribute(
-                SpanAttributes.TRACELOOP_ENTITY_OUTPUT, json.dumps(output)
-            )
+            span.set_attribute(SPAN_OUTPUT, json.dumps(output))
 
     @classmethod
     def set_session(
@@ -391,9 +404,23 @@ class Laminar:
         """
         association_properties = {}
         if session_id is not None:
-            association_properties["session_id"] = session_id
+            association_properties[SESSION_ID] = session_id
         if user_id is not None:
-            association_properties["user_id"] = user_id
+            association_properties[USER_ID] = user_id
+        update_association_properties(association_properties)
+
+    @classmethod
+    def _set_trace_type(
+        cls,
+        trace_type: TraceType,
+    ):
+        """Set the trace_type for the current span and the context
+        Args:
+            trace_type (TraceType): Type of the trace
+        """
+        association_properties = {
+            TRACE_TYPE: trace_type.value,
+        }
         update_association_properties(association_properties)
 
     @classmethod
@@ -405,9 +432,9 @@ class Laminar:
         set_association_properties(props)
 
     @classmethod
-    def create_evaluation(cls, name: str) -> CreateEvaluationResponse:
+    def create_evaluation(cls, name: Optional[str]) -> CreateEvaluationResponse:
         response = requests.post(
-            cls.__base_url + "/v1/evaluations",
+            cls.__base_http_url + "/v1/evaluations",
             data=json.dumps({"name": name}),
             headers=cls._headers(),
         )
@@ -421,14 +448,14 @@ class Laminar:
 
     @classmethod
     def post_evaluation_results(
-        cls, evaluation_name: str, data: list[EvaluationResultDatapoint]
+        cls, evaluation_id: uuid.UUID, data: list[EvaluationResultDatapoint]
     ) -> requests.Response:
         body = {
-            "name": evaluation_name,
-            "points": data,
+            "evaluationId": str(evaluation_id),
+            "points": [datapoint.to_dict() for datapoint in data],
         }
         response = requests.post(
-            cls.__base_url + "/v1/evaluation-datapoints",
+            cls.__base_http_url + "/v1/evaluation-datapoints",
             data=json.dumps(body),
             headers=cls._headers(),
         )
@@ -446,28 +473,38 @@ class Laminar:
 
     @classmethod
     def update_evaluation_status(
-        cls, evaluation_name: str, status: str
-    ) -> requests.Response:
+        cls, evaluation_id: str, status: str
+    ) -> UpdateEvaluationResponse:
+        """
+        Updates the status of an evaluation. Returns the updated evaluation object.
+
+        Args:
+            evaluation_id (str): The ID of the evaluation to update.
+            status (str): The status to set for the evaluation.
+
+        Returns:
+            UpdateEvaluationResponse: The updated evaluation response.
+
+        Raises:
+            ValueError: If the request fails.
+        """
         body = {
-            "name": evaluation_name,
             "status": status,
         }
-        response = requests.put(
-            cls.__base_url + "/v1/evaluations/",
+        url = f"{cls.__base_http_url}/v1/evaluations/{evaluation_id}"
+
+        response = requests.post(
+            url,
             data=json.dumps(body),
             headers=cls._headers(),
         )
         if response.status_code != 200:
-            try:
-                resp_json = response.json()
-                raise ValueError(
-                    f"Failed to send evaluation status. Response: {json.dumps(resp_json)}"
-                )
-            except Exception:
-                raise ValueError(
-                    f"Failed to send evaluation status. Error: {response.text}"
-                )
-        return response
+            raise ValueError(
+                f"Failed to update evaluation status {evaluation_id}. "
+                f"Response: {response.text}"
+            )
+
+        return UpdateEvaluationResponse.model_validate(response.json())
 
     @classmethod
     def _headers(cls):

lmnr/sdk/types.py

@@ -1,10 +1,11 @@
 import datetime
-import requests
+from enum import Enum
 import pydantic
+import requests
+from typing import Any, Awaitable, Callable, Literal, Optional, Union
 import uuid
-from typing import Any, Literal, Optional, TypeAlias, Union
 
-from .utils import to_dict
+from .utils import serialize
 
 
 class ChatMessage(pydantic.BaseModel):
@@ -17,9 +18,11 @@ class ConditionedValue(pydantic.BaseModel):
     value: "NodeInput"
 
 
-Numeric: TypeAlias = Union[int, float]
-NodeInput: TypeAlias = Union[str, list[ChatMessage], ConditionedValue, Numeric, bool]
-PipelineOutput: TypeAlias = Union[NodeInput]
+Numeric = Union[int, float]
+NumericTypes = (int, float)  # for use with isinstance
+
+NodeInput = Union[str, list[ChatMessage], ConditionedValue, Numeric, bool]
+PipelineOutput = Union[NodeInput]
 
 
 class PipelineRunRequest(pydantic.BaseModel):
@@ -35,7 +38,7 @@ class PipelineRunRequest(pydantic.BaseModel):
     def to_dict(self):
         return {
             "inputs": {
-                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else to_dict(v)
+                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
                 for k, v in self.inputs.items()
             },
             "pipeline": self.pipeline,
@@ -74,12 +77,12 @@ class PipelineRunError(Exception):
             return super().__str__()
 
 
-EvaluationDatapointData: TypeAlias = dict[str, Any]
-EvaluationDatapointTarget: TypeAlias = dict[str, Any]
+EvaluationDatapointData = dict[str, Any]
+EvaluationDatapointTarget = dict[str, Any]
 
 
 # EvaluationDatapoint is a single data point in the evaluation
-class EvaluationDatapoint(pydantic.BaseModel):
+class Datapoint(pydantic.BaseModel):
     # input to the executor function. Must be a dict with string keys
     data: EvaluationDatapointData
     # input to the evaluator function (alongside the executor output).
@@ -87,24 +90,24 @@ class EvaluationDatapoint(pydantic.BaseModel):
     target: EvaluationDatapointTarget
 
 
-ExecutorFunctionReturnType: TypeAlias = Any
-EvaluatorFunctionReturnType: TypeAlias = Union[Numeric, dict[str, Numeric]]
+ExecutorFunctionReturnType = Any
+EvaluatorFunctionReturnType = Union[Numeric, dict[str, Numeric]]
 
-# ExecutorFunction: TypeAlias = Callable[
-#     [EvaluationDatapointData, *tuple[Any, ...], dict[str, Any]],
-#     Union[ExecutorFunctionReturnType, Awaitable[ExecutorFunctionReturnType]],
-# ]
+ExecutorFunction = Callable[
+    [EvaluationDatapointData, Any, dict[str, Any]],
+    Union[ExecutorFunctionReturnType, Awaitable[ExecutorFunctionReturnType]],
+]
 
 # EvaluatorFunction is a function that takes the output of the executor and the
 # target data, and returns a score. The score can be a single number or a
 # record of string keys and number values. The latter is useful for evaluating
 # multiple criteria in one go instead of running multiple evaluators.
-# EvaluatorFunction: TypeAlias = Callable[
-#     [ExecutorFunctionReturnType, *tuple[Any, ...], dict[str, Any]],
-#     Union[EvaluatorFunctionReturnType, Awaitable[EvaluatorFunctionReturnType]],
-# ]
+EvaluatorFunction = Callable[
+    [ExecutorFunctionReturnType, Any, dict[str, Any]],
+    Union[EvaluatorFunctionReturnType, Awaitable[EvaluatorFunctionReturnType]],
+]
 
-EvaluationStatus: TypeAlias = Literal["Started", "Finished", "Error"]
+EvaluationStatus = Literal["Started", "Finished", "Error"]
 
 
 class CreateEvaluationResponse(pydantic.BaseModel):
@@ -114,6 +117,10 @@ class CreateEvaluationResponse(pydantic.BaseModel):
     status: EvaluationStatus
     projectId: uuid.UUID
     metadata: Optional[dict[str, Any]] = None
+    averageScores: Optional[dict[str, Numeric]] = None
+
+
+UpdateEvaluationResponse = CreateEvaluationResponse
 
 
 class EvaluationResultDatapoint(pydantic.BaseModel):
@@ -121,3 +128,35 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     target: EvaluationDatapointTarget
     executor_output: ExecutorFunctionReturnType
     scores: dict[str, Numeric]
+    trace_id: uuid.UUID
+
+    # uuid is not serializable by default, so we need to convert it to a string
+    def to_dict(self):
+        return {
+            "data": {
+                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
+                for k, v in self.data.items()
+            },
+            "target": {
+                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
+                for k, v in self.target.items()
+            },
+            "executorOutput": serialize(self.executor_output),
+            "scores": self.scores,
+            "traceId": str(self.trace_id),
+        }
+
+
+class SpanType(Enum):
+    DEFAULT = "DEFAULT"
+    LLM = "LLM"
+    PIPELINE = "PIPELINE"  # must not be set manually
+    EXECUTOR = "EXECUTOR"
+    EVALUATOR = "EVALUATOR"
+    EVALUATION = "EVALUATION"
+
+
+class TraceType(Enum):
+    DEFAULT = "DEFAULT"
+    EVENT = "EVENT"  # must not be set manually
+    EVALUATION = "EVALUATION"

lmnr/sdk/utils.py

@@ -1,5 +1,4 @@
 import asyncio
-import copy
 import datetime
 import dataclasses
 import enum
@@ -50,7 +49,7 @@ def is_iterator(o: typing.Any) -> bool:
     return hasattr(o, "__iter__") and hasattr(o, "__next__")
 
 
-def to_dict(obj: typing.Any) -> dict[str, typing.Any]:
+def serialize(obj: typing.Any) -> dict[str, typing.Any]:
     def to_dict_inner(o: typing.Any):
         if isinstance(o, (datetime.datetime, datetime.date)):
             return o.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
@@ -59,7 +58,7 @@ def to_dict(obj: typing.Any) -> dict[str, typing.Any]:
         elif isinstance(o, (int, float, str, bool)):
             return o
         elif isinstance(o, uuid.UUID):
-            return str(o)  # same as in return, but explicit
+            return str(o)  # same as in final return, but explicit
         elif isinstance(o, enum.Enum):
             return o.value
         elif dataclasses.is_dataclass(o):
@@ -90,11 +89,11 @@ def get_input_from_func_args(
 ) -> dict[str, typing.Any]:
     # Remove implicitly passed "self" or "cls" argument for
     # instance or class methods
-    res = copy.deepcopy(func_kwargs)
+    res = func_kwargs.copy()
     for i, k in enumerate(inspect.signature(func).parameters.keys()):
         if is_method and k in ["self", "cls"]:
             continue
         # If param has default value, then it's not present in func args
-        if len(func_args) > i:
+        if i < len(func_args):
             res[k] = func_args[i]
     return res

lmnr/traceloop_sdk/__init__.py

@@ -3,20 +3,16 @@ import sys
 from pathlib import Path
 
 from typing import Optional, Set
-from colorama import Fore
 from opentelemetry.sdk.trace import SpanProcessor
 from opentelemetry.sdk.trace.export import SpanExporter
-from opentelemetry.sdk.metrics.export import MetricExporter
 from opentelemetry.sdk.resources import SERVICE_NAME
 from opentelemetry.propagators.textmap import TextMapPropagator
 from opentelemetry.util.re import parse_env_headers
 
-from lmnr.traceloop_sdk.metrics.metrics import MetricsWrapper
 from lmnr.traceloop_sdk.instruments import Instruments
 from lmnr.traceloop_sdk.config import (
     is_content_tracing_enabled,
     is_tracing_enabled,
-    is_metrics_enabled,
 )
 from lmnr.traceloop_sdk.tracing.tracing import TracerWrapper
 from typing import Dict
@@ -38,8 +34,6 @@ class Traceloop:
         headers: Dict[str, str] = {},
         disable_batch=False,
         exporter: Optional[SpanExporter] = None,
-        metrics_exporter: Optional[MetricExporter] = None,
-        metrics_headers: Optional[Dict[str, str]] = None,
         processor: Optional[SpanProcessor] = None,
         propagator: Optional[TextMapPropagator] = None,
         should_enrich_metrics: bool = True,
@@ -50,14 +44,11 @@ class Traceloop:
         api_key = os.getenv("TRACELOOP_API_KEY") or api_key
 
         if not is_tracing_enabled():
-            print(Fore.YELLOW + "Tracing is disabled" + Fore.RESET)
+            # print(Fore.YELLOW + "Tracing is disabled" + Fore.RESET)
             return
 
         enable_content_tracing = is_content_tracing_enabled()
 
-        if exporter or processor:
-            print(Fore.GREEN + "Laminar exporting traces to a custom exporter")
-
         headers = os.getenv("TRACELOOP_HEADERS") or headers
 
         if isinstance(headers, str):
@@ -70,30 +61,18 @@ class Traceloop:
             and not api_key
         ):
             print(
-                Fore.RED
-                + "Error: Missing API key,"
+                "Error: Missing API key,"
                 + " go to project settings to create one"
             )
             print("Set the LMNR_PROJECT_API_KEY environment variable to the key")
-            print(Fore.RESET)
             return
 
-        if not exporter and not processor and headers:
-            print(
-                Fore.GREEN
-                + f"Laminar exporting traces to {api_endpoint}, authenticating with custom headers"
-            )
-
         if api_key and not exporter and not processor and not headers:
-            print(
-                Fore.GREEN
-                + f"Laminar exporting traces to {api_endpoint} authenticating with bearer token"
-            )
             headers = {
                 "Authorization": f"Bearer {api_key}",
             }
 
-        print(Fore.RESET)
+        # print(Fore.RESET)
 
         # Tracer init
         resource_attributes.update({SERVICE_NAME: app_name})
@@ -108,21 +87,3 @@ class Traceloop:
             should_enrich_metrics=should_enrich_metrics,
             instruments=instruments,
         )
-
-        if not metrics_exporter and exporter:
-            return
-
-        metrics_endpoint = os.getenv("TRACELOOP_METRICS_ENDPOINT") or api_endpoint
-        metrics_headers = (
-            os.getenv("TRACELOOP_METRICS_HEADERS") or metrics_headers or headers
-        )
-
-        if not is_metrics_enabled() or not metrics_exporter and exporter:
-            print(Fore.YELLOW + "Metrics are disabled" + Fore.RESET)
-            return
-
-        MetricsWrapper.set_static_params(
-            resource_attributes, metrics_endpoint, metrics_headers
-        )
-
-        Traceloop.__metrics_wrapper = MetricsWrapper(exporter=metrics_exporter)

lmnr/traceloop_sdk/config/__init__.py

@@ -7,7 +7,3 @@ def is_tracing_enabled() -> bool:
 
 def is_content_tracing_enabled() -> bool:
     return (os.getenv("TRACELOOP_TRACE_CONTENT") or "true").lower() == "true"
-
-
-def is_metrics_enabled() -> bool:
-    return (os.getenv("TRACELOOP_METRICS_ENABLED") or "true").lower() == "true"

lmnr/traceloop_sdk/decorators/base.py

@@ -7,9 +7,10 @@ import warnings
 
 from opentelemetry import trace
 from opentelemetry import context as context_api
-from opentelemetry.semconv_ai import SpanAttributes
 
+from lmnr.sdk.utils import get_input_from_func_args, is_method
 from lmnr.traceloop_sdk.tracing import get_tracer
+from lmnr.traceloop_sdk.tracing.attributes import SPAN_INPUT, SPAN_OUTPUT
 from lmnr.traceloop_sdk.tracing.tracing import TracerWrapper
 from lmnr.traceloop_sdk.utils.json_encoder import JSONEncoder
 
@@ -52,8 +53,12 @@ def entity_method(
                 try:
                     if _should_send_prompts():
                         span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_INPUT,
-                            _json_dumps({"args": args, "kwargs": kwargs}),
+                            SPAN_INPUT,
+                            _json_dumps(
+                                get_input_from_func_args(
+                                    fn, is_method(fn), args, kwargs
+                                )
+                            ),
                         )
                 except TypeError:
                     pass
@@ -67,7 +72,7 @@ def entity_method(
                 try:
                     if _should_send_prompts():
                         span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_OUTPUT,
+                            SPAN_OUTPUT,
                             _json_dumps(res),
                         )
                 except TypeError:
@@ -105,8 +110,12 @@ def aentity_method(
                 try:
                     if _should_send_prompts():
                         span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_INPUT,
-                            _json_dumps({"args": args, "kwargs": kwargs}),
+                            SPAN_INPUT,
+                            _json_dumps(
+                                get_input_from_func_args(
+                                    fn, is_method(fn), args, kwargs
+                                )
+                            ),
                         )
                 except TypeError:
                     pass
@@ -119,9 +128,7 @@ def aentity_method(
 
                 try:
                     if _should_send_prompts():
-                        span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_OUTPUT, json.dumps(res)
-                        )
+                        span.set_attribute(SPAN_OUTPUT, json.dumps(res))
                 except TypeError:
                     pass
 

lmnr/traceloop_sdk/instruments.py

@@ -2,6 +2,8 @@ from enum import Enum
 
 
 class Instruments(Enum):
+    # The list of libraries which will be autoinstrumented
+    # if no specific instruments are provided to initialize()
     OPENAI = "openai"
     ANTHROPIC = "anthropic"
     COHERE = "cohere"
@@ -15,10 +17,6 @@ class Instruments(Enum):
     MILVUS = "milvus"
     TRANSFORMERS = "transformers"
     TOGETHER = "together"
-    REDIS = "redis"
-    REQUESTS = "requests"
-    URLLIB3 = "urllib3"
-    PYMYSQL = "pymysql"
     BEDROCK = "bedrock"
     REPLICATE = "replicate"
     VERTEXAI = "vertexai"
@@ -27,3 +25,10 @@ class Instruments(Enum):
     ALEPHALPHA = "alephalpha"
     MARQO = "marqo"
     LANCEDB = "lancedb"
+
+    # The following libraries will not be autoinstrumented unless
+    # specified explicitly in the initialize() call.
+    REDIS = "redis"
+    REQUESTS = "requests"
+    URLLIB3 = "urllib3"
+    PYMYSQL = "pymysql"

lmnr/traceloop_sdk/tracing/attributes.py

@@ -0,0 +1,8 @@
+SPAN_INPUT = "lmnr.span.input"
+SPAN_OUTPUT = "lmnr.span.output"
+SPAN_TYPE = "lmnr.span.type"
+
+ASSOCIATION_PROPERTIES = "lmnr.association.properties"
+SESSION_ID = "session_id"
+USER_ID = "user_id"
+TRACE_TYPE = "trace_type"