lmnr 0.4.37__py3-none-any.whl → 0.4.45__py3-none-any.whl

lmnr/__init__.py

@@ -7,6 +7,7 @@ from .sdk.types import (
     NodeInput,
     PipelineRunError,
     PipelineRunResponse,
+    TracingLevel,
 )
 from .sdk.decorators import observe
 from .openllmetry_sdk import Instruments

lmnr/cli.py

@@ -1,26 +1,34 @@
 from argparse import ArgumentParser
 import asyncio
-import importlib
+import importlib.util
 import os
 import sys
 
-from lmnr.sdk.evaluations import set_global_evaluation
+from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
 
 
-# TODO: Refactor this code
 async def run_evaluation(args):
-    sys.path.insert(0, os.getcwd())
+    sys.path.append(os.getcwd())
 
-    with set_global_evaluation(True):
+    prep_token = PREPARE_ONLY.set(True)
+    try:
         file = os.path.abspath(args.file)
+        name = "user_module"
 
-        spec = importlib.util.spec_from_file_location("run_eval", file)
+        spec = importlib.util.spec_from_file_location(name, file)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Could not load module specification from {file}")
         mod = importlib.util.module_from_spec(spec)
+        sys.modules[name] = mod
+
         spec.loader.exec_module(mod)
+        evaluation = EVALUATION_INSTANCE.get()
+        if evaluation is None:
+            raise RuntimeError("Evaluation instance not found")
 
-        from lmnr.sdk.evaluations import _evaluation
-        evaluation = _evaluation
         await evaluation.run()
+    finally:
+        PREPARE_ONLY.reset(prep_token)
 
 
 def cli():
@@ -31,9 +39,15 @@ def cli():
 
     subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
 
-    parser_eval = subparsers.add_parser("eval", description="Run an evaluation")
+    parser_eval = subparsers.add_parser(
+        "eval",
+        description="Run an evaluation",
+        help="Run an evaluation",
+    )
     parser_eval.add_argument("file", help="A file containing the evaluation to run")
-    parser_eval.set_defaults(func=run_evaluation)
 
     parsed = parser.parse_args()
-    asyncio.run(parsed.func(parsed))
+    if parsed.subcommand == "eval":
+        asyncio.run(run_evaluation(parsed))
+    else:
+        parser.print_help()

lmnr/openllmetry_sdk/decorators/base.py

@@ -3,11 +3,13 @@ from functools import wraps
 import logging
 import os
 import pydantic
+import traceback
 import types
 from typing import Any, Optional
 
 from opentelemetry import trace
 from opentelemetry import context as context_api
+from opentelemetry.trace import Span
 
 from lmnr.sdk.utils import get_input_from_func_args, is_method
 from lmnr.openllmetry_sdk.tracing import get_tracer
@@ -69,7 +71,12 @@ def entity_method(
                 except TypeError:
                     pass
 
-                res = fn(*args, **kwargs)
+                try:
+                    res = fn(*args, **kwargs)
+                except Exception as e:
+                    _process_exception(span, e)
+                    span.end()
+                    raise e
 
                 # span will be ended in the generator
                 if isinstance(res, types.GeneratorType):
@@ -131,7 +138,12 @@ def aentity_method(
                 except TypeError:
                     pass
 
-                res = await fn(*args, **kwargs)
+                try:
+                    res = await fn(*args, **kwargs)
+                except Exception as e:
+                    _process_exception(span, e)
+                    span.end()
+                    raise e
 
                 # span will be ended in the generator
                 if isinstance(res, types.AsyncGeneratorType):
@@ -177,3 +189,17 @@ def _should_send_prompts():
     return (
         os.getenv("TRACELOOP_TRACE_CONTENT") or "true"
     ).lower() == "true" or context_api.get_value("override_enable_content_tracing")
+
+
+def _process_exception(span: Span, e: Exception):
+    exception_path = [type(e).__module__] if type(e).__module__ != "builtins" else []
+    exception_path.append(type(e).__qualname__)
+    span.add_event(
+        "exception",
+        {
+            "exception.message": str(e),
+            "exception.type": ".".join(exception_path),
+            "exception.stacktrace": traceback.format_exc(),
+            "exception.escaped": True,
+        },
+    )

lmnr/openllmetry_sdk/tracing/attributes.py

@@ -12,6 +12,7 @@ ASSOCIATION_PROPERTIES = "lmnr.association.properties"
 SESSION_ID = "session_id"
 USER_ID = "user_id"
 TRACE_TYPE = "trace_type"
+TRACING_LEVEL = "tracing_level"
 
 
 # exposed to the user, configurable

lmnr/openllmetry_sdk/tracing/tracing.py

@@ -10,6 +10,7 @@ from lmnr.openllmetry_sdk.tracing.attributes import (
     ASSOCIATION_PROPERTIES,
     SPAN_INSTRUMENTATION_SOURCE,
     SPAN_PATH,
+    TRACING_LEVEL,
 )
 from lmnr.openllmetry_sdk.tracing.content_allow_list import ContentAllowList
 from lmnr.openllmetry_sdk.utils import is_notebook
@@ -183,11 +184,7 @@ class TracerWrapper(object):
 
     @classmethod
     def verify_initialized(cls) -> bool:
-        if hasattr(cls, "instance"):
-            return True
-
-        cls.__logger.warning("Laminar not initialized, make sure to initialize")
-        return False
+        return hasattr(cls, "instance")
 
     def flush(self):
         self.__spans_processor.force_flush()
@@ -228,6 +225,9 @@ def remove_association_properties(properties: dict) -> None:
 
 def _set_association_properties_attributes(span, properties: dict) -> None:
     for key, value in properties.items():
+        if key == TRACING_LEVEL:
+            span.set_attribute(f"lmnr.internal.{TRACING_LEVEL}", value)
+            continue
         span.set_attribute(f"{ASSOCIATION_PROPERTIES}.{key}", value)
 
 
@@ -407,7 +407,9 @@ def init_instrumentations(
 
 def init_openai_instrumentor(should_enrich_metrics: bool):
     try:
-        if is_package_installed("openai"):
+        if is_package_installed("openai") and is_package_installed(
+            "opentelemetry-instrumentation-openai"
+        ):
             from opentelemetry.instrumentation.openai import OpenAIInstrumentor
 
             instrumentor = OpenAIInstrumentor(
@@ -425,7 +427,9 @@ def init_openai_instrumentor(should_enrich_metrics: bool):
 
 def init_anthropic_instrumentor(should_enrich_metrics: bool):
     try:
-        if is_package_installed("anthropic"):
+        if is_package_installed("anthropic") and is_package_installed(
+            "opentelemetry-instrumentation-anthropic"
+        ):
             from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor
 
             instrumentor = AnthropicInstrumentor(
@@ -442,7 +446,9 @@ def init_anthropic_instrumentor(should_enrich_metrics: bool):
 
 def init_cohere_instrumentor():
     try:
-        if is_package_installed("cohere"):
+        if is_package_installed("cohere") and is_package_installed(
+            "opentelemetry-instrumentation-cohere"
+        ):
             from opentelemetry.instrumentation.cohere import CohereInstrumentor
 
             instrumentor = CohereInstrumentor()
@@ -456,7 +462,9 @@ def init_cohere_instrumentor():
 
 def init_pinecone_instrumentor():
     try:
-        if is_package_installed("pinecone"):
+        if is_package_installed("pinecone") and is_package_installed(
+            "opentelemetry-instrumentation-pinecone"
+        ):
             from opentelemetry.instrumentation.pinecone import PineconeInstrumentor
 
             instrumentor = PineconeInstrumentor()
@@ -470,7 +478,9 @@ def init_pinecone_instrumentor():
 
 def init_qdrant_instrumentor():
     try:
-        if is_package_installed("qdrant_client"):
+        if is_package_installed("qdrant_client") and is_package_installed(
+            "opentelemetry-instrumentation-qdrant"
+        ):
             from opentelemetry.instrumentation.qdrant import QdrantInstrumentor
 
             instrumentor = QdrantInstrumentor()
@@ -483,7 +493,9 @@ def init_qdrant_instrumentor():
 
 def init_chroma_instrumentor():
     try:
-        if is_package_installed("chromadb"):
+        if is_package_installed("chromadb") and is_package_installed(
+            "opentelemetry-instrumentation-chromadb"
+        ):
             from opentelemetry.instrumentation.chromadb import ChromaInstrumentor
 
             instrumentor = ChromaInstrumentor()
@@ -497,7 +509,9 @@ def init_chroma_instrumentor():
 
 def init_google_generativeai_instrumentor():
     try:
-        if is_package_installed("google.generativeai"):
+        if is_package_installed("google.generativeai") and is_package_installed(
+            "opentelemetry-instrumentation-google-generativeai"
+        ):
             from opentelemetry.instrumentation.google_generativeai import (
                 GoogleGenerativeAiInstrumentor,
             )
@@ -513,7 +527,9 @@ def init_google_generativeai_instrumentor():
 
 def init_haystack_instrumentor():
     try:
-        if is_package_installed("haystack"):
+        if is_package_installed("haystack") and is_package_installed(
+            "opentelemetry-instrumentation-haystack"
+        ):
             from opentelemetry.instrumentation.haystack import HaystackInstrumentor
 
             instrumentor = HaystackInstrumentor()
@@ -527,7 +543,9 @@ def init_haystack_instrumentor():
 
 def init_langchain_instrumentor():
     try:
-        if is_package_installed("langchain"):
+        if is_package_installed("langchain") and is_package_installed(
+            "opentelemetry-instrumentation-langchain"
+        ):
             from opentelemetry.instrumentation.langchain import LangchainInstrumentor
 
             instrumentor = LangchainInstrumentor()
@@ -543,7 +561,9 @@ def init_langchain_instrumentor():
 
 def init_mistralai_instrumentor():
     try:
-        if is_package_installed("mistralai"):
+        if is_package_installed("mistralai") and is_package_installed(
+            "opentelemetry-instrumentation-mistralai"
+        ):
             from opentelemetry.instrumentation.mistralai import MistralAiInstrumentor
 
             instrumentor = MistralAiInstrumentor()
@@ -557,7 +577,9 @@ def init_mistralai_instrumentor():
 
 def init_ollama_instrumentor():
     try:
-        if is_package_installed("ollama"):
+        if is_package_installed("ollama") and is_package_installed(
+            "opentelemetry-instrumentation-ollama"
+        ):
             from opentelemetry.instrumentation.ollama import OllamaInstrumentor
 
             instrumentor = OllamaInstrumentor()
@@ -571,7 +593,9 @@ def init_ollama_instrumentor():
 
 def init_transformers_instrumentor():
     try:
-        if is_package_installed("transformers"):
+        if is_package_installed("transformers") and is_package_installed(
+            "opentelemetry-instrumentation-transformers"
+        ):
             from opentelemetry.instrumentation.transformers import (
                 TransformersInstrumentor,
             )
@@ -587,7 +611,9 @@ def init_transformers_instrumentor():
 
 def init_together_instrumentor():
     try:
-        if is_package_installed("together"):
+        if is_package_installed("together") and is_package_installed(
+            "opentelemetry-instrumentation-together"
+        ):
             from opentelemetry.instrumentation.together import TogetherAiInstrumentor
 
             instrumentor = TogetherAiInstrumentor()
@@ -601,7 +627,9 @@ def init_together_instrumentor():
 
 def init_llama_index_instrumentor():
     try:
-        if is_package_installed("llama-index") or is_package_installed("llama_index"):
+        if (
+            is_package_installed("llama-index") or is_package_installed("llama_index")
+        ) and is_package_installed("opentelemetry-instrumentation-llamaindex"):
             from opentelemetry.instrumentation.llamaindex import LlamaIndexInstrumentor
 
             instrumentor = LlamaIndexInstrumentor()
@@ -615,7 +643,9 @@ def init_llama_index_instrumentor():
 
 def init_milvus_instrumentor():
     try:
-        if is_package_installed("pymilvus"):
+        if is_package_installed("pymilvus") and is_package_installed(
+            "opentelemetry-instrumentation-milvus"
+        ):
             from opentelemetry.instrumentation.milvus import MilvusInstrumentor
 
             instrumentor = MilvusInstrumentor()
@@ -671,7 +701,9 @@ def init_pymysql_instrumentor():
 
 def init_bedrock_instrumentor(should_enrich_metrics: bool):
     try:
-        if is_package_installed("boto3"):
+        if is_package_installed("boto3") and is_package_installed(
+            "opentelemetry-instrumentation-bedrock"
+        ):
             from opentelemetry.instrumentation.bedrock import BedrockInstrumentor
 
             instrumentor = BedrockInstrumentor(
@@ -687,7 +719,9 @@ def init_bedrock_instrumentor(should_enrich_metrics: bool):
 
 def init_replicate_instrumentor():
     try:
-        if is_package_installed("replicate"):
+        if is_package_installed("replicate") and is_package_installed(
+            "opentelemetry-instrumentation-replicate"
+        ):
             from opentelemetry.instrumentation.replicate import ReplicateInstrumentor
 
             instrumentor = ReplicateInstrumentor()
@@ -701,7 +735,9 @@ def init_replicate_instrumentor():
 
 def init_vertexai_instrumentor():
     try:
-        if is_package_installed("vertexai"):
+        if is_package_installed("vertexai") and is_package_installed(
+            "opentelemetry-instrumentation-vertexai"
+        ):
             from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
 
             instrumentor = VertexAIInstrumentor()
@@ -715,9 +751,10 @@ def init_vertexai_instrumentor():
 
 def init_watsonx_instrumentor():
     try:
-        if is_package_installed("ibm-watsonx-ai") or is_package_installed(
-            "ibm-watson-machine-learning"
-        ):
+        if (
+            is_package_installed("ibm-watsonx-ai")
+            or is_package_installed("ibm-watson-machine-learning")
+        ) and is_package_installed("opentelemetry-instrumentation-watsonx"):
             from opentelemetry.instrumentation.watsonx import WatsonxInstrumentor
 
             instrumentor = WatsonxInstrumentor()
@@ -731,7 +768,9 @@ def init_watsonx_instrumentor():
 
 def init_weaviate_instrumentor():
     try:
-        if is_package_installed("weaviate"):
+        if is_package_installed("weaviate") and is_package_installed(
+            "opentelemetry-instrumentation-weaviate"
+        ):
             from opentelemetry.instrumentation.weaviate import WeaviateInstrumentor
 
             instrumentor = WeaviateInstrumentor()
@@ -745,7 +784,9 @@ def init_weaviate_instrumentor():
 
 def init_alephalpha_instrumentor():
     try:
-        if is_package_installed("aleph_alpha_client"):
+        if is_package_installed("aleph_alpha_client") and is_package_installed(
+            "opentelemetry-instrumentation-alephalpha"
+        ):
             from opentelemetry.instrumentation.alephalpha import AlephAlphaInstrumentor
 
             instrumentor = AlephAlphaInstrumentor()
@@ -759,7 +800,9 @@ def init_alephalpha_instrumentor():
 
 def init_marqo_instrumentor():
     try:
-        if is_package_installed("marqo"):
+        if is_package_installed("marqo") and is_package_installed(
+            "opentelemetry-instrumentation-marqo"
+        ):
             from opentelemetry.instrumentation.marqo import MarqoInstrumentor
 
             instrumentor = MarqoInstrumentor()
@@ -773,7 +816,9 @@ def init_marqo_instrumentor():
 
 def init_lancedb_instrumentor():
     try:
-        if is_package_installed("lancedb"):
+        if is_package_installed("lancedb") and is_package_installed(
+            "opentelemetry-instrumentation-lancedb"
+        ):
             from opentelemetry.instrumentation.lancedb import LanceInstrumentor
 
             instrumentor = LanceInstrumentor()
@@ -786,7 +831,9 @@ def init_lancedb_instrumentor():
 
 def init_redis_instrumentor():
     try:
-        if is_package_installed("redis"):
+        if is_package_installed("redis") and is_package_installed(
+            "opentelemetry-instrumentation-redis"
+        ):
             from opentelemetry.instrumentation.redis import RedisInstrumentor
 
             instrumentor = RedisInstrumentor()
@@ -800,7 +847,9 @@ def init_redis_instrumentor():
 
 def init_groq_instrumentor():
     try:
-        if is_package_installed("groq"):
+        if is_package_installed("groq") and is_package_installed(
+            "opentelemetry-instrumentation-groq"
+        ):
             from opentelemetry.instrumentation.groq import GroqInstrumentor
 
             instrumentor = GroqInstrumentor()
@@ -814,7 +863,9 @@ def init_groq_instrumentor():
 
 def init_sagemaker_instrumentor(should_enrich_metrics: bool):
     try:
-        if is_package_installed("boto3"):
+        if is_package_installed("boto3") and is_package_installed(
+            "opentelemetry-instrumentation-sagemaker"
+        ):
             from opentelemetry.instrumentation.sagemaker import SageMakerInstrumentor
 
             instrumentor = SageMakerInstrumentor(

lmnr/openllmetry_sdk/utils/package_check.py

@@ -2,5 +2,6 @@ from importlib.metadata import distributions
 
 installed_packages = {dist.metadata["Name"].lower() for dist in distributions()}
 
+
 def is_package_installed(package_name: str) -> bool:
     return package_name.lower() in installed_packages

lmnr/sdk/datasets.py

@@ -1,11 +1,9 @@
 from abc import ABC, abstractmethod
-import logging
+import asyncio
 
 from .log import get_default_logger
 from .laminar import Laminar as L
-from .types import (
-    Datapoint,
-)
+from .types import Datapoint
 
 DEFAULT_FETCH_SIZE = 25
 

lmnr/sdk/decorators.py

@@ -7,7 +7,7 @@ from opentelemetry.trace import INVALID_SPAN, get_current_span
 from typing import Callable, Optional, TypeVar, cast
 from typing_extensions import ParamSpec
 
-from lmnr.openllmetry_sdk.tracing.attributes import SESSION_ID, USER_ID
+from lmnr.openllmetry_sdk.tracing.attributes import SESSION_ID
 from lmnr.openllmetry_sdk.tracing.tracing import update_association_properties
 
 from .utils import is_async
@@ -20,7 +20,6 @@ R = TypeVar("R")
 def observe(
     *,
     name: Optional[str] = None,
-    user_id: Optional[str] = None,
     session_id: Optional[str] = None,
 ) -> Callable[[Callable[P, R]], Callable[P, R]]:
     """The main decorator entrypoint for Laminar. This is used to wrap
@@ -30,9 +29,6 @@ def observe(
         name (Optional[str], optional): Name of the span. Function
                         name is used if not specified.
                         Defaults to None.
-        user_id (Optional[str], optional): User ID to associate
-                        with the span and the following context.
-                        Defaults to None.
         session_id (Optional[str], optional): Session ID to associate with the
                         span and the following context. Defaults to None.
 
@@ -49,13 +45,9 @@ def observe(
         if current_span != INVALID_SPAN:
             if session_id is not None:
                 current_span.set_attribute(SESSION_ID, session_id)
-            if user_id is not None:
-                current_span.set_attribute(USER_ID, user_id)
         association_properties = {}
         if session_id is not None:
             association_properties["session_id"] = session_id
-        if user_id is not None:
-            association_properties["user_id"] = user_id
         update_association_properties(association_properties)
         return (
             aentity_method(name=name)(func)

lmnr/sdk/eval_control.py

@@ -0,0 +1,4 @@
+from contextvars import ContextVar
+
+PREPARE_ONLY = ContextVar("__lmnr_prepare_only", default=False)
+EVALUATION_INSTANCE = ContextVar("__lmnr_evaluation_instance", default=None)

lmnr/sdk/evaluations.py

@@ -3,7 +3,6 @@ import re
 import sys
 import uuid
 
-from contextlib import contextmanager
 from tqdm import tqdm
 from typing import Any, Awaitable, Optional, Set, Union
 
@@ -11,6 +10,7 @@ from ..openllmetry_sdk.instruments import Instruments
 from ..openllmetry_sdk.tracing.attributes import SPAN_TYPE
 
 from .datasets import EvaluationDataset
+from .eval_control import EVALUATION_INSTANCE, PREPARE_ONLY
 from .laminar import Laminar as L
 from .log import get_default_logger
 from .types import (
@@ -28,21 +28,6 @@ from .utils import is_async
 
 DEFAULT_BATCH_SIZE = 5
 
-_evaluation = None
-_set_global_evaluation = False
-
-
-@contextmanager
-def set_global_evaluation(set_global_evaluation: bool):
-    global _set_global_evaluation
-    original = _set_global_evaluation
-    try:
-        _set_global_evaluation = set_global_evaluation
-        yield
-    finally:
-        _set_global_evaluation = original
-        pass
-
 
 def get_evaluation_url(project_id: str, evaluation_id: str):
     return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
@@ -198,15 +183,10 @@ class Evaluation:
             instruments=instruments,
         )
 
-    def run(self) -> Union[None, Awaitable[None]]:
+    async def run(self) -> Awaitable[None]:
         if self.is_finished:
             raise Exception("Evaluation is already finished")
-
-        loop = asyncio.get_event_loop()
-        if loop.is_running():
-            return loop.create_task(self._run())
-        else:
-            return loop.run_until_complete(self._run())
+        return await self._run()
 
     async def _run(self) -> None:
         self.reporter.start(len(self.data))
@@ -224,7 +204,7 @@ class Evaluation:
         for result_datapoint in result_datapoints:
             result_datapoint.human_evaluators = self.human_evaluators or {}
 
-        evaluation = L.create_evaluation(
+        evaluation = await L.create_evaluation(
             data=result_datapoints, group_id=self.group_id, name=self.name
         )
         average_scores = get_average_scores(result_datapoints)
@@ -389,8 +369,11 @@ def evaluate(
         instruments=instruments,
     )
 
-    global _evaluation
-    if _set_global_evaluation:
-        _evaluation = evaluation
+    if PREPARE_ONLY.get():
+        EVALUATION_INSTANCE.set(evaluation)
     else:
-        return evaluation.run()
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            return loop.run_until_complete(evaluation.run())
+        else:
+            return asyncio.run(evaluation.run())