lmnr 0.4.12b3__py3-none-any.whl → 0.4.12b4__py3-none-any.whl

lmnr/sdk/decorators.py

@@ -6,6 +6,7 @@ from opentelemetry.trace import INVALID_SPAN, get_current_span
 
 from typing import Callable, Optional, cast
 
+from lmnr.traceloop_sdk.tracing.attributes import SESSION_ID, USER_ID
 from lmnr.traceloop_sdk.tracing.tracing import update_association_properties
 
 from .utils import is_async
@@ -43,11 +44,11 @@ def observe(
         if current_span != INVALID_SPAN:
             if session_id is not None:
                 current_span.set_attribute(
-                    "traceloop.association.properties.session_id", session_id
+                    SESSION_ID, session_id
                 )
             if user_id is not None:
                 current_span.set_attribute(
-                    "traceloop.association.properties.user_id", user_id
+                    USER_ID, user_id
                 )
         association_properties = {}
         if session_id is not None:

lmnr/sdk/evaluations.py

@@ -2,12 +2,26 @@ import asyncio
 import sys
 from abc import ABC, abstractmethod
 from contextlib import contextmanager
-from typing import Any, Awaitable, Optional, Union
+from typing import Any, Awaitable, Optional, Set, Union
+import uuid
 
 from tqdm import tqdm
 
+from ..traceloop_sdk.instruments import Instruments
+from ..traceloop_sdk.tracing.attributes import SPAN_TYPE
+
 from .laminar import Laminar as L
-from .types import CreateEvaluationResponse, Datapoint, EvaluationResultDatapoint, Numeric, NumericTypes
+from .types import (
+    CreateEvaluationResponse,
+    Datapoint,
+    EvaluationResultDatapoint,
+    EvaluatorFunction,
+    ExecutorFunction,
+    Numeric,
+    NumericTypes,
+    SpanType,
+    TraceType,
+)
 from .utils import is_async
 
 DEFAULT_BATCH_SIZE = 5
@@ -39,7 +53,11 @@ class EvaluationReporter:
     def start(self, name: str, project_id: str, id: str, length: int):
         print(f"Running evaluation {name}...\n")
         print(f"Check progress and results at {get_evaluation_url(project_id, id)}\n")
-        self.cli_progress = tqdm(total=length, bar_format="{bar} {percentage:3.0f}% | ETA: {remaining}s | {n_fmt}/{total_fmt}", ncols=60)
+        self.cli_progress = tqdm(
+            total=length,
+            bar_format="{bar} {percentage:3.0f}% | ETA: {remaining}s | {n_fmt}/{total_fmt}",
+            ncols=60,
+        )
 
     def update(self, batch_length: int):
         self.cli_progress.update(batch_length)
@@ -51,7 +69,7 @@ class EvaluationReporter:
     def stop(self, average_scores: dict[str, Numeric]):
         self.cli_progress.close()
         print("\nAverage scores:")
-        for (name, score) in average_scores.items():
+        for name, score in average_scores.items():
             print(f"{name}: {score}")
         print("\n")
 
@@ -78,12 +96,14 @@ class Evaluation:
         self,
         data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
         executor: Any,
-        evaluators: list[Any],
+        evaluators: dict[str, EvaluatorFunction],
         name: Optional[str] = None,
         batch_size: int = DEFAULT_BATCH_SIZE,
         project_api_key: Optional[str] = None,
         base_url: Optional[str] = None,
         http_port: Optional[int] = None,
+        grpc_port: Optional[int] = None,
+        instruments: Optional[Set[Instruments]] = None,
     ):
         """
         Initializes an instance of the Evaluations class.
@@ -114,33 +134,18 @@ class Evaluation:
                             Defaults to "https://api.lmnr.ai".
             http_port (Optional[int], optional): The port for the Laminar API HTTP service.
                             Defaults to 443.
+            instruments (Optional[Set[Instruments]], optional): Set of modules to auto-instrument.
+                            Defaults to None. If None, all available instruments will be used.
         """
 
         self.is_finished = False
         self.name = name
         self.reporter = EvaluationReporter()
         self.executor = executor
-        self.evaluators = dict(
-            zip(
-                [
-                    (
-                        e.__name__
-                        if e.__name__ and e.__name__ != "<lambda>"
-                        else f"evaluator_{i+1}"
-                    )
-                    for i, e in enumerate(evaluators)
-                ],
-                evaluators,
-            )
-        )
-        self.evaluator_names = list(self.evaluators.keys())
+        self.evaluators = evaluators
         if isinstance(data, list):
             self.data = [
-                (
-                    Datapoint.model_validate(point)
-                    if isinstance(point, dict)
-                    else point
-                )
+                (Datapoint.model_validate(point) if isinstance(point, dict) else point)
                 for point in data
             ]
         else:
@@ -150,7 +155,8 @@ class Evaluation:
             project_api_key=project_api_key,
             base_url=base_url,
             http_port=http_port,
-            instruments=set(),
+            grpc_port=grpc_port,
+            instruments=instruments,
         )
 
     def run(self) -> Union[None, Awaitable[None]]:
@@ -205,7 +211,7 @@ class Evaluation:
     async def evaluate_in_batches(self, evaluation: CreateEvaluationResponse):
         for i in range(0, len(self.data), self.batch_size):
             batch = (
-                self.data[i: i + self.batch_size]
+                self.data[i : i + self.batch_size]
                 if isinstance(self.data, list)
                 else self.data.slice(i, i + self.batch_size)
             )
@@ -217,52 +223,72 @@ class Evaluation:
             finally:
                 self.reporter.update(len(batch))
 
-    async def _evaluate_batch(self, batch: list[Datapoint]) -> list[EvaluationResultDatapoint]:
+    async def _evaluate_batch(
+        self, batch: list[Datapoint]
+    ) -> list[EvaluationResultDatapoint]:
         batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
         results = await asyncio.gather(*batch_promises)
         return results
 
-    async def _evaluate_datapoint(self, datapoint) -> EvaluationResultDatapoint:
-        output = (
-            await self.executor(datapoint.data)
-            if is_async(self.executor)
-            else self.executor(datapoint.data)
-        )
-        target = datapoint.target
-
-        # Iterate over evaluators
-        scores: dict[str, Numeric] = {}
-        for evaluator_name in self.evaluator_names:
-            evaluator = self.evaluators[evaluator_name]
-            value = (
-                await evaluator(output, target)
-                if is_async(evaluator)
-                else evaluator(output, target)
+    async def _evaluate_datapoint(
+        self, datapoint: Datapoint
+    ) -> EvaluationResultDatapoint:
+        with L.start_as_current_span("evaluation") as evaluation_span:
+            L._set_trace_type(trace_type=TraceType.EVALUATION)
+            evaluation_span.set_attribute(SPAN_TYPE, SpanType.EVALUATION.value)
+            with L.start_as_current_span(
+                "executor", input={"data": datapoint.data}
+            ) as executor_span:
+                executor_span.set_attribute(SPAN_TYPE, SpanType.EXECUTOR.value)
+                output = (
+                    await self.executor(datapoint.data)
+                    if is_async(self.executor)
+                    else self.executor(datapoint.data)
+                )
+                L.set_span_output(output)
+            target = datapoint.target
+
+            # Iterate over evaluators
+            scores: dict[str, Numeric] = {}
+            for evaluator_name, evaluator in self.evaluators.items():
+                with L.start_as_current_span(
+                    "evaluator", input={"output": output, "target": target}
+                ) as evaluator_span:
+                    evaluator_span.set_attribute(SPAN_TYPE, SpanType.EVALUATOR.value)
+                    value = (
+                        await evaluator(output, target)
+                        if is_async(evaluator)
+                        else evaluator(output, target)
+                    )
+                    L.set_span_output(value)
+
+                # If evaluator returns a single number, use evaluator name as key
+                if isinstance(value, NumericTypes):
+                    scores[evaluator_name] = value
+                else:
+                    scores.update(value)
+
+            trace_id = uuid.UUID(int=evaluation_span.get_span_context().trace_id)
+            return EvaluationResultDatapoint(
+                data=datapoint.data,
+                target=target,
+                executor_output=output,
+                scores=scores,
+                trace_id=trace_id,
             )
 
-            # If evaluator returns a single number, use evaluator name as key
-            if isinstance(value, NumericTypes):
-                scores[evaluator_name] = value
-            else:
-                scores.update(value)
-
-        return EvaluationResultDatapoint(
-            data=datapoint.data,
-            target=target,
-            executorOutput=output,
-            scores=scores,
-        )
-
 
 def evaluate(
     data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
-    executor: Any,
-    evaluators: list[Any],
+    executor: ExecutorFunction,
+    evaluators: dict[str, EvaluatorFunction],
     name: Optional[str] = None,
     batch_size: int = DEFAULT_BATCH_SIZE,
     project_api_key: Optional[str] = None,
     base_url: Optional[str] = None,
     http_port: Optional[int] = None,
+    grpc_port: Optional[int] = None,
+    instruments: Optional[Set[Instruments]] = None,
 ) -> Optional[Awaitable[None]]:
     """
     If added to the file which is called through lmnr eval command, then simply registers the evaluation.
@@ -295,6 +321,10 @@ def evaluate(
                         Defaults to "https://api.lmnr.ai".
         http_port (Optional[int], optional): The port for the Laminar API HTTP service.
                         Defaults to 443.
+        grpc_port (Optional[int], optional): The port for the Laminar API gRPC service.
+                        Defaults to 8443.
+        instruments (Optional[Set[Instruments]], optional): Set of modules to auto-instrument.
+                        Defaults to None. If None, all available instruments will be used.
     """
 
     evaluation = Evaluation(
@@ -306,6 +336,8 @@ def evaluate(
         project_api_key=project_api_key,
         base_url=base_url,
         http_port=http_port,
+        grpc_port=grpc_port,
+        instruments=instruments,
     )
 
     global _evaluation

lmnr/sdk/laminar.py

@@ -5,7 +5,6 @@ from opentelemetry.trace import (
     get_current_span,
     SpanKind,
 )
-from opentelemetry.semconv_ai import SpanAttributes
 from opentelemetry.util.types import AttributeValue
 from opentelemetry.context.context import Context
 from opentelemetry.util import types
@@ -26,7 +25,17 @@ import os
 import requests
 import uuid
 
-from lmnr.traceloop_sdk.tracing.tracing import set_association_properties, update_association_properties
+from lmnr.traceloop_sdk.tracing.attributes import (
+    SESSION_ID,
+    SPAN_INPUT,
+    SPAN_OUTPUT,
+    TRACE_TYPE,
+    USER_ID,
+)
+from lmnr.traceloop_sdk.tracing.tracing import (
+    set_association_properties,
+    update_association_properties,
+)
 
 from .log import VerboseColorfulFormatter
 
@@ -37,6 +46,7 @@ from .types import (
     PipelineRunResponse,
     NodeInput,
     PipelineRunRequest,
+    TraceType,
     UpdateEvaluationResponse,
 )
 
@@ -356,8 +366,8 @@ class Laminar:
             ) as span:
                 if input is not None:
                     span.set_attribute(
-                        SpanAttributes.TRACELOOP_ENTITY_INPUT,
-                        json.dumps({"input": input}),
+                        SPAN_INPUT,
+                        json.dumps(input),
                     )
                 yield span
 
@@ -371,9 +381,7 @@ class Laminar:
         """
         span = get_current_span()
         if output is not None and span != INVALID_SPAN:
-            span.set_attribute(
-                SpanAttributes.TRACELOOP_ENTITY_OUTPUT, json.dumps(output)
-            )
+            span.set_attribute(SPAN_OUTPUT, json.dumps(output))
 
     @classmethod
     def set_session(
@@ -396,9 +404,23 @@ class Laminar:
         """
         association_properties = {}
         if session_id is not None:
-            association_properties["session_id"] = session_id
+            association_properties[SESSION_ID] = session_id
         if user_id is not None:
-            association_properties["user_id"] = user_id
+            association_properties[USER_ID] = user_id
+        update_association_properties(association_properties)
+
+    @classmethod
+    def _set_trace_type(
+        cls,
+        trace_type: TraceType,
+    ):
+        """Set the trace_type for the current span and the context
+        Args:
+            trace_type (TraceType): Type of the trace
+        """
+        association_properties = {
+            TRACE_TYPE: trace_type.value,
+        }
         update_association_properties(association_properties)
 
     @classmethod
@@ -430,7 +452,7 @@ class Laminar:
     ) -> requests.Response:
         body = {
             "evaluationId": str(evaluation_id),
-            "points": [datapoint.model_dump() for datapoint in data],
+            "points": [datapoint.to_dict() for datapoint in data],
         }
         response = requests.post(
             cls.__base_http_url + "/v1/evaluation-datapoints",

lmnr/sdk/types.py

@@ -1,10 +1,11 @@
 import datetime
-import requests
+from enum import Enum
 import pydantic
-import uuid
+import requests
 from typing import Any, Awaitable, Callable, Literal, Optional, Union
+import uuid
 
-from .utils import to_dict
+from .utils import serialize
 
 
 class ChatMessage(pydantic.BaseModel):
@@ -37,7 +38,7 @@ class PipelineRunRequest(pydantic.BaseModel):
     def to_dict(self):
         return {
             "inputs": {
-                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else to_dict(v)
+                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
                 for k, v in self.inputs.items()
             },
             "pipeline": self.pipeline,
@@ -125,5 +126,37 @@ UpdateEvaluationResponse = CreateEvaluationResponse
 class EvaluationResultDatapoint(pydantic.BaseModel):
     data: EvaluationDatapointData
     target: EvaluationDatapointTarget
-    executorOutput: ExecutorFunctionReturnType
+    executor_output: ExecutorFunctionReturnType
     scores: dict[str, Numeric]
+    trace_id: uuid.UUID
+
+    # uuid is not serializable by default, so we need to convert it to a string
+    def to_dict(self):
+        return {
+            "data": {
+                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
+                for k, v in self.data.items()
+            },
+            "target": {
+                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
+                for k, v in self.target.items()
+            },
+            "executorOutput": serialize(self.executor_output),
+            "scores": self.scores,
+            "traceId": str(self.trace_id),
+        }
+
+
+class SpanType(Enum):
+    DEFAULT = "DEFAULT"
+    LLM = "LLM"
+    PIPELINE = "PIPELINE"  # must not be set manually
+    EXECUTOR = "EXECUTOR"
+    EVALUATOR = "EVALUATOR"
+    EVALUATION = "EVALUATION"
+
+
+class TraceType(Enum):
+    DEFAULT = "DEFAULT"
+    EVENT = "EVENT"  # must not be set manually
+    EVALUATION = "EVALUATION"

lmnr/sdk/utils.py

@@ -1,5 +1,4 @@
 import asyncio
-import copy
 import datetime
 import dataclasses
 import enum
@@ -50,7 +49,7 @@ def is_iterator(o: typing.Any) -> bool:
     return hasattr(o, "__iter__") and hasattr(o, "__next__")
 
 
-def to_dict(obj: typing.Any) -> dict[str, typing.Any]:
+def serialize(obj: typing.Any) -> dict[str, typing.Any]:
     def to_dict_inner(o: typing.Any):
         if isinstance(o, (datetime.datetime, datetime.date)):
             return o.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
@@ -59,7 +58,7 @@ def to_dict(obj: typing.Any) -> dict[str, typing.Any]:
         elif isinstance(o, (int, float, str, bool)):
             return o
         elif isinstance(o, uuid.UUID):
-            return str(o)  # same as in return, but explicit
+            return str(o)  # same as in final return, but explicit
         elif isinstance(o, enum.Enum):
             return o.value
         elif dataclasses.is_dataclass(o):
@@ -90,11 +89,11 @@ def get_input_from_func_args(
 ) -> dict[str, typing.Any]:
     # Remove implicitly passed "self" or "cls" argument for
     # instance or class methods
-    res = copy.deepcopy(func_kwargs)
+    res = func_kwargs.copy()
     for i, k in enumerate(inspect.signature(func).parameters.keys()):
         if is_method and k in ["self", "cls"]:
             continue
         # If param has default value, then it's not present in func args
-        if len(func_args) > i:
+        if i < len(func_args):
             res[k] = func_args[i]
     return res

lmnr/traceloop_sdk/__init__.py

@@ -3,20 +3,16 @@ import sys
 from pathlib import Path
 
 from typing import Optional, Set
-from colorama import Fore
 from opentelemetry.sdk.trace import SpanProcessor
 from opentelemetry.sdk.trace.export import SpanExporter
-from opentelemetry.sdk.metrics.export import MetricExporter
 from opentelemetry.sdk.resources import SERVICE_NAME
 from opentelemetry.propagators.textmap import TextMapPropagator
 from opentelemetry.util.re import parse_env_headers
 
-from lmnr.traceloop_sdk.metrics.metrics import MetricsWrapper
 from lmnr.traceloop_sdk.instruments import Instruments
 from lmnr.traceloop_sdk.config import (
     is_content_tracing_enabled,
     is_tracing_enabled,
-    is_metrics_enabled,
 )
 from lmnr.traceloop_sdk.tracing.tracing import TracerWrapper
 from typing import Dict
@@ -38,8 +34,6 @@ class Traceloop:
         headers: Dict[str, str] = {},
         disable_batch=False,
         exporter: Optional[SpanExporter] = None,
-        metrics_exporter: Optional[MetricExporter] = None,
-        metrics_headers: Optional[Dict[str, str]] = None,
         processor: Optional[SpanProcessor] = None,
         propagator: Optional[TextMapPropagator] = None,
         should_enrich_metrics: bool = True,
@@ -50,7 +44,7 @@ class Traceloop:
         api_key = os.getenv("TRACELOOP_API_KEY") or api_key
 
         if not is_tracing_enabled():
-            print(Fore.YELLOW + "Tracing is disabled" + Fore.RESET)
+            # print(Fore.YELLOW + "Tracing is disabled" + Fore.RESET)
             return
 
         enable_content_tracing = is_content_tracing_enabled()
@@ -67,12 +61,10 @@ class Traceloop:
             and not api_key
         ):
             print(
-                Fore.RED
-                + "Error: Missing API key,"
+                "Error: Missing API key,"
                 + " go to project settings to create one"
             )
             print("Set the LMNR_PROJECT_API_KEY environment variable to the key")
-            print(Fore.RESET)
             return
 
         if api_key and not exporter and not processor and not headers:
@@ -80,7 +72,7 @@ class Traceloop:
                 "Authorization": f"Bearer {api_key}",
             }
 
-        print(Fore.RESET)
+        # print(Fore.RESET)
 
         # Tracer init
         resource_attributes.update({SERVICE_NAME: app_name})
@@ -95,21 +87,3 @@ class Traceloop:
             should_enrich_metrics=should_enrich_metrics,
             instruments=instruments,
         )
-
-        if not metrics_exporter and exporter:
-            return
-
-        metrics_endpoint = os.getenv("TRACELOOP_METRICS_ENDPOINT") or api_endpoint
-        metrics_headers = (
-            os.getenv("TRACELOOP_METRICS_HEADERS") or metrics_headers or headers
-        )
-
-        if not is_metrics_enabled() or not metrics_exporter and exporter:
-            print(Fore.YELLOW + "Metrics are disabled" + Fore.RESET)
-            return
-
-        MetricsWrapper.set_static_params(
-            resource_attributes, metrics_endpoint, metrics_headers
-        )
-
-        Traceloop.__metrics_wrapper = MetricsWrapper(exporter=metrics_exporter)

lmnr/traceloop_sdk/config/__init__.py

@@ -7,7 +7,3 @@ def is_tracing_enabled() -> bool:
 
 def is_content_tracing_enabled() -> bool:
     return (os.getenv("TRACELOOP_TRACE_CONTENT") or "true").lower() == "true"
-
-
-def is_metrics_enabled() -> bool:
-    return (os.getenv("TRACELOOP_METRICS_ENABLED") or "true").lower() == "true"

lmnr/traceloop_sdk/decorators/base.py

@@ -7,9 +7,10 @@ import warnings
 
 from opentelemetry import trace
 from opentelemetry import context as context_api
-from opentelemetry.semconv_ai import SpanAttributes
 
+from lmnr.sdk.utils import get_input_from_func_args, is_method
 from lmnr.traceloop_sdk.tracing import get_tracer
+from lmnr.traceloop_sdk.tracing.attributes import SPAN_INPUT, SPAN_OUTPUT
 from lmnr.traceloop_sdk.tracing.tracing import TracerWrapper
 from lmnr.traceloop_sdk.utils.json_encoder import JSONEncoder
 
@@ -52,8 +53,12 @@ def entity_method(
                 try:
                     if _should_send_prompts():
                         span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_INPUT,
-                            _json_dumps({"args": args, "kwargs": kwargs}),
+                            SPAN_INPUT,
+                            _json_dumps(
+                                get_input_from_func_args(
+                                    fn, is_method(fn), args, kwargs
+                                )
+                            ),
                         )
                 except TypeError:
                     pass
@@ -67,7 +72,7 @@ def entity_method(
                 try:
                     if _should_send_prompts():
                         span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_OUTPUT,
+                            SPAN_OUTPUT,
                             _json_dumps(res),
                         )
                 except TypeError:
@@ -105,8 +110,12 @@ def aentity_method(
                 try:
                     if _should_send_prompts():
                         span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_INPUT,
-                            _json_dumps({"args": args, "kwargs": kwargs}),
+                            SPAN_INPUT,
+                            _json_dumps(
+                                get_input_from_func_args(
+                                    fn, is_method(fn), args, kwargs
+                                )
+                            ),
                         )
                 except TypeError:
                     pass
@@ -119,9 +128,7 @@ def aentity_method(
 
                 try:
                     if _should_send_prompts():
-                        span.set_attribute(
-                            SpanAttributes.TRACELOOP_ENTITY_OUTPUT, json.dumps(res)
-                        )
+                        span.set_attribute(SPAN_OUTPUT, json.dumps(res))
                 except TypeError:
                     pass
 

lmnr/traceloop_sdk/tracing/attributes.py

@@ -0,0 +1,8 @@
+SPAN_INPUT = "lmnr.span.input"
+SPAN_OUTPUT = "lmnr.span.output"
+SPAN_TYPE = "lmnr.span.type"
+
+ASSOCIATION_PROPERTIES = "lmnr.association.properties"
+SESSION_ID = "session_id"
+USER_ID = "user_id"
+TRACE_TYPE = "trace_type"

lmnr/traceloop_sdk/tracing/tracing.py

@@ -3,7 +3,6 @@ import logging
 import os
 
 
-from colorama import Fore
 from opentelemetry import trace
 from opentelemetry.exporter.otlp.proto.http.trace_exporter import (
     OTLPSpanExporter as HTTPExporter,
@@ -24,17 +23,16 @@ from opentelemetry.trace import get_tracer_provider, ProxyTracerProvider
 from opentelemetry.context import get_value, attach, set_value
 from opentelemetry.instrumentation.threading import ThreadingInstrumentor
 
-from opentelemetry.semconv_ai import SpanAttributes
-
 # from lmnr.traceloop_sdk import Telemetry
 from lmnr.traceloop_sdk.instruments import Instruments
+from lmnr.traceloop_sdk.tracing.attributes import ASSOCIATION_PROPERTIES
 from lmnr.traceloop_sdk.tracing.content_allow_list import ContentAllowList
 from lmnr.traceloop_sdk.utils import is_notebook
 from lmnr.traceloop_sdk.utils.package_check import is_package_installed
 from typing import Dict, Optional, Set
 
 
-TRACER_NAME = "traceloop.tracer"
+TRACER_NAME = "lmnr.tracer"
 EXCLUDED_URLS = """
     iam.cloud.ibm.com,
     dataplatform.cloud.ibm.com,
@@ -44,7 +42,7 @@ EXCLUDED_URLS = """
     api.anthropic.com,
     api.cohere.ai,
     pinecone.io,
-    traceloop.com,
+    api.lmnr.ai,
     posthog.com,
     sentry.io,
     bedrock-runtime,
@@ -130,138 +128,103 @@ class TracerWrapper(object):
                 for instrument in instruments:
                     if instrument == Instruments.OPENAI:
                         if not init_openai_instrumentor(should_enrich_metrics):
-                            print(Fore.RED + "Warning: OpenAI library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: OpenAI library does not exist.")
                     elif instrument == Instruments.ANTHROPIC:
                         if not init_anthropic_instrumentor(should_enrich_metrics):
                             print(
-                                Fore.RED + "Warning: Anthropic library does not exist."
+                                 "Warning: Anthropic library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.COHERE:
                         if not init_cohere_instrumentor():
-                            print(Fore.RED + "Warning: Cohere library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: Cohere library does not exist.")
                     elif instrument == Instruments.PINECONE:
                         if not init_pinecone_instrumentor():
                             print(
-                                Fore.RED + "Warning: Pinecone library does not exist."
+                                "Warning: Pinecone library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.CHROMA:
                         if not init_chroma_instrumentor():
-                            print(Fore.RED + "Warning: Chroma library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: Chroma library does not exist.")
                     elif instrument == Instruments.GOOGLE_GENERATIVEAI:
                         if not init_google_generativeai_instrumentor():
-                            print(
-                                Fore.RED
-                                + "Warning: Google Generative AI library does not exist."
-                            )
-                            print(Fore.RESET)
+                            print("Warning: Google Generative AI library does not exist.")
                     elif instrument == Instruments.LANGCHAIN:
                         if not init_langchain_instrumentor():
                             print(
-                                Fore.RED + "Warning: LangChain library does not exist."
+                                "Warning: LangChain library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.MISTRAL:
                         if not init_mistralai_instrumentor():
                             print(
-                                Fore.RED + "Warning: MistralAI library does not exist."
+                                "Warning: MistralAI library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.OLLAMA:
                         if not init_ollama_instrumentor():
-                            print(Fore.RED + "Warning: Ollama library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: Ollama library does not exist.")
                     elif instrument == Instruments.LLAMA_INDEX:
                         if not init_llama_index_instrumentor():
                             print(
-                                Fore.RED + "Warning: LlamaIndex library does not exist."
+                                "Warning: LlamaIndex library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.MILVUS:
                         if not init_milvus_instrumentor():
-                            print(Fore.RED + "Warning: Milvus library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: Milvus library does not exist.")
                     elif instrument == Instruments.TRANSFORMERS:
                         if not init_transformers_instrumentor():
-                            print(
-                                Fore.RED
-                                + "Warning: Transformers library does not exist."
-                            )
-                            print(Fore.RESET)
+                            print("Warning: Transformers library does not exist.")
                     elif instrument == Instruments.TOGETHER:
                         if not init_together_instrumentor():
                             print(
-                                Fore.RED + "Warning: TogetherAI library does not exist."
+                                "Warning: TogetherAI library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.REQUESTS:
                         if not init_requests_instrumentor():
                             print(
-                                Fore.RED + "Warning: Requests library does not exist."
+                                "Warning: Requests library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.URLLIB3:
                         if not init_urllib3_instrumentor():
-                            print(Fore.RED + "Warning: urllib3 library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: urllib3 library does not exist.")
                     elif instrument == Instruments.PYMYSQL:
                         if not init_pymysql_instrumentor():
-                            print(Fore.RED + "Warning: PyMySQL library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: PyMySQL library does not exist.")
                     elif instrument == Instruments.BEDROCK:
                         if not init_bedrock_instrumentor(should_enrich_metrics):
-                            print(Fore.RED + "Warning: Bedrock library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: Bedrock library does not exist.")
                     elif instrument == Instruments.REPLICATE:
                         if not init_replicate_instrumentor():
                             print(
-                                Fore.RED + "Warning: Replicate library does not exist."
+                                "Warning: Replicate library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.VERTEXAI:
                         if not init_vertexai_instrumentor():
                             print(
-                                Fore.RED + "Warning: Vertex AI library does not exist."
+                                "Warning: Vertex AI library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.WATSONX:
                         if not init_watsonx_instrumentor():
-                            print(Fore.RED + "Warning: Watsonx library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: Watsonx library does not exist.")
                     elif instrument == Instruments.WEAVIATE:
                         if not init_weaviate_instrumentor():
                             print(
-                                Fore.RED + "Warning: Weaviate library does not exist."
+                                "Warning: Weaviate library does not exist."
                             )
-                            print(Fore.RESET)
                     elif instrument == Instruments.ALEPHALPHA:
                         if not init_alephalpha_instrumentor():
-                            print(
-                                Fore.RED
-                                + "Warning: Aleph Alpha library does not exist."
-                            )
-                            print(Fore.RESET)
+                            print("Warning: Aleph Alpha library does not exist.")
                     elif instrument == Instruments.MARQO:
                         if not init_marqo_instrumentor():
-                            print(Fore.RED + "Warning: marqo library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: marqo library does not exist.")
                     elif instrument == Instruments.LANCEDB:
                         if not init_lancedb_instrumentor():
-                            print(Fore.RED + "Warning: LanceDB library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: LanceDB library does not exist.")
                     elif instrument == Instruments.REDIS:
                         if not init_redis_instrumentor():
-                            print(Fore.RED + "Warning: redis library does not exist.")
-                            print(Fore.RESET)
+                            print("Warning: redis library does not exist.")
 
                     else:
                         print(
-                            Fore.RED
-                            + "Warning: "
+                            "Warning: "
                             + instrument
                             + " instrumentation does not exist."
                         )
@@ -270,7 +233,6 @@ class TracerWrapper(object):
                             + "from lmnr.traceloop_sdk.instruments import Instruments\n"
                             + 'Traceloop.init(app_name="...", instruments=set([Instruments.OPENAI]))'
                         )
-                        print(Fore.RESET)
 
             obj.__content_allow_list = ContentAllowList()
 
@@ -293,49 +255,6 @@ class TracerWrapper(object):
                 else:
                     attach(set_value("override_enable_content_tracing", False))
 
-        if is_llm_span(span):
-            managed_prompt = get_value("managed_prompt")
-            if managed_prompt is not None:
-                span.set_attribute(
-                    SpanAttributes.TRACELOOP_PROMPT_MANAGED, managed_prompt
-                )
-
-            prompt_key = get_value("prompt_key")
-            if prompt_key is not None:
-                span.set_attribute(SpanAttributes.TRACELOOP_PROMPT_KEY, prompt_key)
-
-            prompt_version = get_value("prompt_version")
-            if prompt_version is not None:
-                span.set_attribute(
-                    SpanAttributes.TRACELOOP_PROMPT_VERSION, prompt_version
-                )
-
-            prompt_version_name = get_value("prompt_version_name")
-            if prompt_version_name is not None:
-                span.set_attribute(
-                    SpanAttributes.TRACELOOP_PROMPT_VERSION_NAME, prompt_version_name
-                )
-
-            prompt_version_hash = get_value("prompt_version_hash")
-            if prompt_version_hash is not None:
-                span.set_attribute(
-                    SpanAttributes.TRACELOOP_PROMPT_VERSION_HASH, prompt_version_hash
-                )
-
-            prompt_template = get_value("prompt_template")
-            if prompt_template is not None:
-                span.set_attribute(
-                    SpanAttributes.TRACELOOP_PROMPT_TEMPLATE, prompt_template
-                )
-
-            prompt_template_variables = get_value("prompt_template_variables")
-            if prompt_template_variables is not None:
-                for key, value in prompt_template_variables.items():
-                    span.set_attribute(
-                        f"{SpanAttributes.TRACELOOP_PROMPT_TEMPLATE_VARIABLES}.{key}",
-                        value,
-                    )
-
         # Call original on_start method if it exists in custom processor
         if self.__spans_processor_original_on_start:
             self.__spans_processor_original_on_start(span, parent_context)
@@ -360,11 +279,7 @@ class TracerWrapper(object):
         if (os.getenv("TRACELOOP_SUPPRESS_WARNINGS") or "false").lower() == "true":
             return False
 
-        print(
-            Fore.RED
-            + "Warning: Traceloop not initialized, make sure you call Traceloop.init()"
-        )
-        print(Fore.RESET)
+        print("Warning: Laminar not initialized, make sure to initialize")
         return False
 
     def flush(self):
@@ -399,7 +314,7 @@ def update_association_properties(properties: dict) -> None:
 def _set_association_properties_attributes(span, properties: dict) -> None:
     for key, value in properties.items():
         span.set_attribute(
-            f"{SpanAttributes.TRACELOOP_ASSOCIATION_PROPERTIES}.{key}", value
+            f"{ASSOCIATION_PROPERTIES}.{key}", value
         )
 
 
@@ -427,10 +342,6 @@ def set_external_prompt_tracing_context(
     attach(set_value("prompt_template_variables", variables))
 
 
-def is_llm_span(span) -> bool:
-    return span.attributes.get(SpanAttributes.LLM_REQUEST_TYPE) is not None
-
-
 def init_spans_exporter(api_endpoint: str, headers: Dict[str, str]) -> SpanExporter:
     if "http" in api_endpoint.lower() or "https" in api_endpoint.lower():
         return HTTPExporter(endpoint=f"{api_endpoint}/v1/traces", headers=headers)
@@ -499,7 +410,6 @@ def init_openai_instrumentor(should_enrich_metrics: bool):
                 # exception_logger=lambda e: Telemetry().log_exception(e),
                 enrich_assistant=should_enrich_metrics,
                 enrich_token_usage=should_enrich_metrics,
-                get_common_metrics_attributes=metrics_common_attributes,
             )
             if not instrumentor.is_instrumented_by_opentelemetry:
                 instrumentor.instrument()
@@ -520,7 +430,6 @@ def init_anthropic_instrumentor(should_enrich_metrics: bool):
             instrumentor = AnthropicInstrumentor(
                 # exception_logger=lambda e: Telemetry().log_exception(e),
                 enrich_token_usage=should_enrich_metrics,
-                get_common_metrics_attributes=metrics_common_attributes,
             )
             if not instrumentor.is_instrumented_by_opentelemetry:
                 instrumentor.instrument()
@@ -988,23 +897,3 @@ def init_groq_instrumentor():
         logging.error(f"Error initializing Groq instrumentor: {e}")
         # Telemetry().log_exception(e)
         return False
-
-
-def metrics_common_attributes():
-    common_attributes = {}
-    workflow_name = get_value("workflow_name")
-    if workflow_name is not None:
-        common_attributes[SpanAttributes.TRACELOOP_WORKFLOW_NAME] = workflow_name
-
-    entity_name = get_value("entity_name")
-    if entity_name is not None:
-        common_attributes[SpanAttributes.TRACELOOP_ENTITY_NAME] = entity_name
-
-    association_properties = get_value("association_properties")
-    if association_properties is not None:
-        for key, value in association_properties.items():
-            common_attributes[
-                f"{SpanAttributes.TRACELOOP_ASSOCIATION_PROPERTIES}.{key}"
-            ] = value
-
-    return common_attributes

{lmnr-0.4.12b3.dist-info → lmnr-0.4.12b4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.12b3
+Version: 0.4.12b4
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -14,7 +14,6 @@ Classifier: Programming Language :: Python :: 3.12
 Requires-Dist: argparse (>=1.0,<2.0)
 Requires-Dist: asyncio (>=3.0,<4.0)
 Requires-Dist: backoff (>=2.0,<3.0)
-Requires-Dist: colorama (>=0.4,<0.5)
 Requires-Dist: deprecated (>=1.0,<2.0)
 Requires-Dist: jinja2 (>=3.0,<4.0)
 Requires-Dist: opentelemetry-api (>=1.27.0,<2.0.0)
@@ -197,7 +196,7 @@ L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"], instruments={In
 
 If you want to fully disable any kind of autoinstrumentation, pass an empty set as `instruments=set()` to `.initialize()`. 
 
-Majority of the autoinstrumentations are provided by Traceloop's [OpenLLMetry](https://github.com/traceloop/openllmetry).
+Autoinstrumentations are provided by Traceloop's [OpenLLMetry](https://github.com/traceloop/openllmetry).
 
 ## Sending events
 
@@ -267,13 +266,14 @@ Evaluation takes in the following parameters:
 - `name` – the name of your evaluation. If no such evaluation exists in the project, it will be created. Otherwise, data will be pushed to the existing evaluation
 - `data` – an array of `EvaluationDatapoint` objects, where each `EvaluationDatapoint` has two keys: `target` and `data`, each containing a key-value object. Alternatively, you can pass in dictionaries, and we will instantiate `EvaluationDatapoint`s with pydantic if possible
 - `executor` – the logic you want to evaluate. This function must take `data` as the first argument, and produce any output. *
-- `evaluators` – evaluaton logic. List of functions that take output of executor as the first argument, `target` as the second argument and produce a numeric scores. Each function can produce either a single number or `dict[str, int|float]` of scores.
+- `evaluators` – evaluaton logic. Functions that take output of executor as the first argument, `target` as the second argument and produce a numeric scores. Pass a dict from evaluator name to a function. Each function can produce either a single number or `dict[str, int|float]` of scores.
 
 \* If you already have the outputs of executors you want to evaluate, you can specify the executor as an identity function, that takes in `data` and returns only needed value(s) from it.
 
-### Example
+### Example code
 
 ```python
+from lmnr import evaluate
 from openai import AsyncOpenAI
 import asyncio
 import os
@@ -304,20 +304,25 @@ data = [
 ]
 
 
-def evaluator_A(output, target):
+def correctness(output, target):
     return 1 if output == target["capital"] else 0
 
 
 # Create an Evaluation instance
-e = Evaluation(
-    name="py-evaluation-async",
+e = evaluate(
+    name="my-evaluation",
     data=data,
     executor=get_capital,
-    evaluators=[evaluator_A],
+    evaluators={"correctness": correctness},
     project_api_key=os.environ["LMNR_PROJECT_API_KEY"],
 )
-
-# Run the evaluation
-asyncio.run(e.run())
 ```
 
+### Running from CLI.
+
+1. Make sure `lmnr` is installed in a venv. CLI does not work with a global env
+1. Run `lmnr path/to/my/eval.py`
+
+### Running from code
+
+Simply execute the function, e.g. `python3 path/to/my/eval.py`

{lmnr-0.4.12b3.dist-info → lmnr-0.4.12b4.dist-info}/RECORD

@@ -1,21 +1,19 @@
 lmnr/__init__.py,sha256=5Ks8UIicCzCBgwSz0MOX3I7jVruPMUO3SmxIwUoODzQ,231
 lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lmnr/sdk/decorators.py,sha256=O8S4PI6LUfdWPkbroigl5khtnkyhp24J8qzSdlvCs44,2227
-lmnr/sdk/evaluations.py,sha256=Ip0XJsdNEPcq4LhPaYF7DOS0y1sDXpPuDf7Vf2YUkJc,11940
-lmnr/sdk/laminar.py,sha256=vG9JkaonK8XBLSFVf4sKV50silbg7UVVLTbWFB3SANw,19011
+lmnr/sdk/decorators.py,sha256=ii7Bqp6flaanIFSK6M1_ZZV-izp4o3hkR1MmY7wnFQQ,2227
+lmnr/sdk/evaluations.py,sha256=kVf6cZAF53HMYIdmlaTV5YD0UdPsv_YCzvs1Mts9Zps,13587
+lmnr/sdk/laminar.py,sha256=3LqzqhsSOHxz11_lxAdvqy_awtOnTdPTeYxYEZ3F4Go,19407
 lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
-lmnr/sdk/types.py,sha256=nSN6Z4CDVFJ0o-vjzIT1qzwRiJ70pReGpqdNOcjXKic,4062
-lmnr/sdk/utils.py,sha256=ZsGJ86tq8lIbvOhSb1gJWH5K3GylO_lgX68FN6rG2nM,3358
+lmnr/sdk/types.py,sha256=QB89q6WeN715x15ukoRVufXk6FSP_1pGn8QsUSIJG5U,5062
+lmnr/sdk/utils.py,sha256=s81p6uJehgJSaLWy3sR5fTpEDH7vzn3i_UujUHChl6M,3346
 lmnr/traceloop_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/traceloop_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
-lmnr/traceloop_sdk/__init__.py,sha256=-wa25NtU7BeTgqjT6rpVIEdaVP4aOjLgbYrc3B3-JaM,3932
-lmnr/traceloop_sdk/config/__init__.py,sha256=EGN3ixOt_ORbMxqaQdLaC14kmO-gyG4mnGJ2GfN-R-E,364
+lmnr/traceloop_sdk/__init__.py,sha256=hp3q1OsFaGgaQCEanJrL38BJN32hWqCNVCSjYpndEsY,2957
+lmnr/traceloop_sdk/config/__init__.py,sha256=DliMGp2NjYAqRFLKpWQPUKjGMHRO8QsVfazBA1qENQ8,248
 lmnr/traceloop_sdk/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lmnr/traceloop_sdk/decorators/base.py,sha256=wcqXF0iVQgRXMyWTcJ5QvL_6q2y_gttwsX8dllmAtWM,4891
+lmnr/traceloop_sdk/decorators/base.py,sha256=5YCzAErlhv1bMDO1C9LBlLWYk3bwku0RLjGLR-TkR4c,5128
 lmnr/traceloop_sdk/instruments.py,sha256=oMvIASueW3GeChpjIdH-DD9aFBVB8OtHZ0HawppTrlI,942
-lmnr/traceloop_sdk/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lmnr/traceloop_sdk/metrics/metrics.py,sha256=AlQ2a2os1WcZbfBd155u_UzBbPrbuPia6O_HbojV9Wc,5055
 lmnr/traceloop_sdk/tests/__init__.py,sha256=RYnG0-8zbXL0-2Ste1mEBf5sN4d_rQjGTCgPBuaZC74,20
 lmnr/traceloop_sdk/tests/cassettes/test_association_properties/test_langchain_and_external_association_properties.yaml,sha256=26g0wRA0juicHg_XrhcE8H4vhs1lawDs0o0aLFn-I7w,3103
 lmnr/traceloop_sdk/tests/cassettes/test_association_properties/test_langchain_association_properties.yaml,sha256=FNlSWlYCsWc3w7UPZzfGjDnxS3gAOhL-kpsu4BTxsDE,3061
@@ -37,17 +35,17 @@ lmnr/traceloop_sdk/tests/test_sdk_initialization.py,sha256=fRaf6lrxFzJIN94P1Tav_
 lmnr/traceloop_sdk/tests/test_tasks.py,sha256=xlEx8BKp4yG83SCjK5WkPGfyC33JSrx4h8VyjVwGbgw,906
 lmnr/traceloop_sdk/tests/test_workflows.py,sha256=RVcfY3WAFIDZC15-aSua21aoQyYeWE7KypDyUsm-2EM,9372
 lmnr/traceloop_sdk/tracing/__init__.py,sha256=Ckq7zCM26VdJVB5tIZv0GTPyMZKyfso_KWD5yPHaqdo,66
+lmnr/traceloop_sdk/tracing/attributes.py,sha256=Rvglt_2IeZzKJ-mumrp9qAtTwHza34CrNgv4CvYihk0,221
 lmnr/traceloop_sdk/tracing/content_allow_list.py,sha256=3feztm6PBWNelc8pAZUcQyEGyeSpNiVKjOaDk65l2ps,846
 lmnr/traceloop_sdk/tracing/context_manager.py,sha256=csVlB6kDmbgSPsROHwnddvGGblx55v6lJMRj0wsSMQM,304
-lmnr/traceloop_sdk/tracing/manual.py,sha256=RPwEreHHdzmw7g15u4G21GqhHOvRp7d72ylQNLG1jRM,1841
-lmnr/traceloop_sdk/tracing/tracing.py,sha256=5e8AsiFKaIO6zqAbMfhw242glVsQUkxbNhTWP7QDqSg,40108
+lmnr/traceloop_sdk/tracing/tracing.py,sha256=8plGdX6nErrPERgYXQDQRyBTtVgv2Ies46ph-msLLQE,35443
 lmnr/traceloop_sdk/utils/__init__.py,sha256=pNhf0G3vTd5ccoc03i1MXDbricSaiqCbi1DLWhSekK8,604
 lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQvzJk_fZH0OOsRAM1iZQXsR8,2112
 lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
 lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
 lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
-lmnr-0.4.12b3.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.12b3.dist-info/METADATA,sha256=VnqchbOU6e4xrMuioUj2uJPWDNWGkcz5nmOM-Fua0UI,11992
-lmnr-0.4.12b3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lmnr-0.4.12b3.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.12b3.dist-info/RECORD,,
+lmnr-0.4.12b4.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.12b4.dist-info/METADATA,sha256=u8pLIuLaw6if3D60lp5ekTKpVoJGxt3Z-gcjVKoDj7g,12196
+lmnr-0.4.12b4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lmnr-0.4.12b4.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.12b4.dist-info/RECORD,,

lmnr/traceloop_sdk/metrics/metrics.py

@@ -1,176 +0,0 @@
-from collections.abc import Sequence
-from typing import Dict
-
-from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
-    OTLPMetricExporter as GRPCExporter,
-)
-from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
-    OTLPMetricExporter as HTTPExporter,
-)
-from opentelemetry.semconv_ai import Meters
-from opentelemetry.sdk.metrics import MeterProvider
-from opentelemetry.sdk.metrics.export import (
-    PeriodicExportingMetricReader,
-    MetricExporter,
-)
-from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
-from opentelemetry.sdk.resources import Resource
-
-from opentelemetry import metrics
-
-
-class MetricsWrapper(object):
-    resource_attributes: dict = {}
-    endpoint: str = None
-    # if it needs headers?
-    headers: Dict[str, str] = {}
-    __metrics_exporter: MetricExporter = None
-    __metrics_provider: MeterProvider = None
-
-    def __new__(cls, exporter: MetricExporter = None) -> "MetricsWrapper":
-        if not hasattr(cls, "instance"):
-            obj = cls.instance = super(MetricsWrapper, cls).__new__(cls)
-            if not MetricsWrapper.endpoint:
-                return obj
-
-            obj.__metrics_exporter = (
-                exporter
-                if exporter
-                else init_metrics_exporter(
-                    MetricsWrapper.endpoint, MetricsWrapper.headers
-                )
-            )
-
-            obj.__metrics_provider = init_metrics_provider(
-                obj.__metrics_exporter, MetricsWrapper.resource_attributes
-            )
-
-        return cls.instance
-
-    @staticmethod
-    def set_static_params(
-        resource_attributes: dict,
-        endpoint: str,
-        headers: Dict[str, str],
-    ) -> None:
-        MetricsWrapper.resource_attributes = resource_attributes
-        MetricsWrapper.endpoint = endpoint
-        MetricsWrapper.headers = headers
-
-
-def init_metrics_exporter(endpoint: str, headers: Dict[str, str]) -> MetricExporter:
-    if "http" in endpoint.lower() or "https" in endpoint.lower():
-        return HTTPExporter(endpoint=f"{endpoint}/v1/metrics", headers=headers)
-    else:
-        return GRPCExporter(endpoint=endpoint, headers=headers)
-
-
-def init_metrics_provider(
-    exporter: MetricExporter, resource_attributes: dict = None
-) -> MeterProvider:
-    resource = (
-        Resource.create(resource_attributes)
-        if resource_attributes
-        else Resource.create()
-    )
-    reader = PeriodicExportingMetricReader(exporter)
-    provider = MeterProvider(
-        metric_readers=[reader],
-        resource=resource,
-        views=metric_views(),
-    )
-
-    metrics.set_meter_provider(provider)
-    return provider
-
-
-def metric_views() -> Sequence[View]:
-    return [
-        View(
-            instrument_name=Meters.LLM_TOKEN_USAGE,
-            aggregation=ExplicitBucketHistogramAggregation(
-                [
-                    0.01,
-                    0.02,
-                    0.04,
-                    0.08,
-                    0.16,
-                    0.32,
-                    0.64,
-                    1.28,
-                    2.56,
-                    5.12,
-                    10.24,
-                    20.48,
-                    40.96,
-                    81.92,
-                ]
-            ),
-        ),
-        View(
-            instrument_name=Meters.LLM_OPERATION_DURATION,
-            aggregation=ExplicitBucketHistogramAggregation(
-                [
-                    1,
-                    4,
-                    16,
-                    64,
-                    256,
-                    1024,
-                    4096,
-                    16384,
-                    65536,
-                    262144,
-                    1048576,
-                    4194304,
-                    16777216,
-                    67108864,
-                ]
-            ),
-        ),
-        View(
-            instrument_name=Meters.PINECONE_DB_QUERY_DURATION,
-            aggregation=ExplicitBucketHistogramAggregation(
-                [
-                    0.01,
-                    0.02,
-                    0.04,
-                    0.08,
-                    0.16,
-                    0.32,
-                    0.64,
-                    1.28,
-                    2.56,
-                    5.12,
-                    10.24,
-                    20.48,
-                    40.96,
-                    81.92,
-                ]
-            ),
-        ),
-        View(
-            instrument_name=Meters.PINECONE_DB_QUERY_SCORES,
-            aggregation=ExplicitBucketHistogramAggregation(
-                [
-                    -1,
-                    -0.875,
-                    -0.75,
-                    -0.625,
-                    -0.5,
-                    -0.375,
-                    -0.25,
-                    -0.125,
-                    0,
-                    0.125,
-                    0.25,
-                    0.375,
-                    0.5,
-                    0.625,
-                    0.75,
-                    0.875,
-                    1,
-                ]
-            ),
-        ),
-    ]

lmnr/traceloop_sdk/tracing/manual.py

@@ -1,57 +0,0 @@
-from contextlib import contextmanager
-from opentelemetry.semconv_ai import SpanAttributes
-from opentelemetry.trace import Span
-from pydantic import BaseModel
-from lmnr.traceloop_sdk.tracing.context_manager import get_tracer
-
-
-class LLMMessage(BaseModel):
-    role: str
-    content: str
-
-
-class LLMUsage(BaseModel):
-    prompt_tokens: int
-    completion_tokens: int
-    total_tokens: int
-
-
-class LLMSpan:
-    _span: Span = None
-
-    def __init__(self, span: Span):
-        self._span = span
-        pass
-
-    def report_request(self, model: str, messages: list[LLMMessage]):
-        self._span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, model)
-        for idx, message in enumerate(messages):
-            self._span.set_attribute(
-                f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", message.role
-            )
-            self._span.set_attribute(
-                f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", message.content
-            )
-
-    def report_response(self, model: str, completions: list[str]):
-        self._span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL, model)
-        for idx, completion in enumerate(completions):
-            self._span.set_attribute(
-                f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", "assistant"
-            )
-            self._span.set_attribute(
-                f"{SpanAttributes.LLM_COMPLETIONS}.{idx}", completion
-            )
-
-
-@contextmanager
-def track_llm_call(vendor: str, type: str):
-    with get_tracer() as tracer:
-        with tracer.start_as_current_span(name=f"{vendor}.{type}") as span:
-            span.set_attribute(SpanAttributes.LLM_SYSTEM, vendor)
-            span.set_attribute(SpanAttributes.LLM_REQUEST_TYPE, type)
-            llm_span = LLMSpan(span)
-            try:
-                yield llm_span
-            finally:
-                span.end()