lmnr 0.5.2__py3-none-any.whl → 0.6.0__py3-none-any.whl

lmnr/__init__.py

@@ -13,8 +13,10 @@ from .sdk.types import (
 )
 from .sdk.decorators import observe
 from .sdk.types import LaminarSpanContext
-from .openllmetry_sdk import Instruments
-from .openllmetry_sdk.tracing.attributes import Attributes
+from .opentelemetry_lib.tracing.attributes import Attributes
+from .opentelemetry_lib.tracing.instruments import Instruments
+from .opentelemetry_lib.tracing.processor import LaminarSpanProcessor
+from .opentelemetry_lib.tracing.tracer import get_laminar_tracer_provider, get_tracer
 from opentelemetry.trace import use_span
 
 __all__ = [
@@ -29,9 +31,12 @@ __all__ = [
     "LaminarClient",
     "LaminarDataset",
     "LaminarSpanContext",
+    "LaminarSpanProcessor",
     "RunAgentResponseChunk",
     "StepChunkContent",
     "TracingLevel",
+    "get_laminar_tracer_provider",
+    "get_tracer",
     "evaluate",
     "observe",
     "use_span",

lmnr/cli.py

@@ -1,18 +1,17 @@
 from argparse import ArgumentParser
 import asyncio
 import importlib.util
-import logging
 import os
 import re
 import sys
+from typing import Optional
+
+from lmnr.sdk.evaluations import Evaluation
 
 from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
-from .sdk.log import ColorfulFormatter
+from .sdk.log import get_default_logger
 
-LOG = logging.getLogger(__name__)
-console_log_handler = logging.StreamHandler()
-console_log_handler.setFormatter(ColorfulFormatter())
-LOG.addHandler(console_log_handler)
+LOG = get_default_logger(__name__)
 
 
 EVAL_DIR = "evals"
@@ -28,7 +27,10 @@ async def run_evaluation(args):
             if re.match(r".*_eval\.py$", f) or re.match(r"eval_.*\.py$", f)
         ]
         if len(files) == 0:
-            LOG.error("No evaluation files found in evals directory")
+            LOG.error("No evaluation files found in `evals` directory")
+            LOG.info(
+                "Eval files must be located in the `evals` directory and must be named *_eval.py or eval_*.py"
+            )
             return
         files.sort()
         LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
@@ -53,7 +55,7 @@ async def run_evaluation(args):
             sys.modules[name] = mod
 
             spec.loader.exec_module(mod)
-            evaluation = EVALUATION_INSTANCE.get()
+            evaluation: Optional[Evaluation] = EVALUATION_INSTANCE.get()
             if evaluation is None:
                 LOG.warning("Evaluation instance not found")
                 if args.fail_on_error:

lmnr/opentelemetry_lib/__init__.py

@@ -0,0 +1,55 @@
+import sys
+
+from typing import Optional, Set
+from opentelemetry.sdk.trace.export import SpanExporter
+from opentelemetry.sdk.resources import SERVICE_NAME
+
+from lmnr.opentelemetry_lib.tracing.instruments import Instruments
+from lmnr.opentelemetry_lib.tracing import TracerWrapper
+
+MAX_MANUAL_SPAN_PAYLOAD_SIZE = 1024 * 1024  # 1MB
+
+
+class TracerManager:
+    __tracer_wrapper: TracerWrapper
+
+    @staticmethod
+    def init(
+        app_name: Optional[str] = sys.argv[0],
+        disable_batch=False,
+        exporter: Optional[SpanExporter] = None,
+        resource_attributes: dict = {},
+        instruments: Optional[Set[Instruments]] = None,
+        base_url: str = "https://api.lmnr.ai",
+        port: int = 8443,
+        http_port: int = 443,
+        project_api_key: Optional[str] = None,
+        max_export_batch_size: Optional[int] = None,
+        force_http: bool = False,
+        timeout_seconds: int = 30,
+    ) -> None:
+        enable_content_tracing = True
+
+        # Tracer init
+        resource_attributes.update({SERVICE_NAME: app_name})
+        TracerWrapper.set_static_params(resource_attributes, enable_content_tracing)
+        TracerManager.__tracer_wrapper = TracerWrapper(
+            disable_batch=disable_batch,
+            exporter=exporter,
+            instruments=instruments,
+            base_url=base_url,
+            port=port,
+            http_port=http_port,
+            project_api_key=project_api_key,
+            max_export_batch_size=max_export_batch_size,
+            force_http=force_http,
+            timeout_seconds=timeout_seconds,
+        )
+
+    @staticmethod
+    def flush() -> bool:
+        return TracerManager.__tracer_wrapper.flush()
+
+    @staticmethod
+    def shutdown():
+        TracerManager.__tracer_wrapper.shutdown()

lmnr/{openllmetry_sdk/decorators/base.py → opentelemetry_lib/decorators/__init__.py}

@@ -1,5 +1,5 @@
-import json
 from functools import wraps
+import json
 import logging
 import pydantic
 import types
@@ -10,11 +10,11 @@ from opentelemetry import context as context_api
 from opentelemetry.trace import Span
 
 from lmnr.sdk.utils import get_input_from_func_args, is_method
-from lmnr.openllmetry_sdk.tracing import get_tracer
-from lmnr.openllmetry_sdk.tracing.attributes import SPAN_INPUT, SPAN_OUTPUT, SPAN_TYPE
-from lmnr.openllmetry_sdk.tracing.tracing import TracerWrapper
-from lmnr.openllmetry_sdk.utils.json_encoder import JSONEncoder
-from lmnr.openllmetry_sdk.config import MAX_MANUAL_SPAN_PAYLOAD_SIZE
+from lmnr.opentelemetry_lib import MAX_MANUAL_SPAN_PAYLOAD_SIZE
+from lmnr.opentelemetry_lib.tracing.tracer import get_tracer
+from lmnr.opentelemetry_lib.tracing.attributes import SPAN_INPUT, SPAN_OUTPUT, SPAN_TYPE
+from lmnr.opentelemetry_lib.tracing import TracerWrapper
+from lmnr.opentelemetry_lib.utils.json_encoder import JSONEncoder
 
 
 class CustomJSONEncoder(JSONEncoder):
@@ -86,7 +86,16 @@ def entity_method(
 
                 # span will be ended in the generator
                 if isinstance(res, types.GeneratorType):
-                    return _handle_generator(span, res)
+                    return _handle_generator(span, ctx_token, res)
+                if isinstance(res, types.AsyncGeneratorType):
+                    # async def foo() -> AsyncGenerator[int, None]:
+                    # is not considered async in a classical sense in Python,
+                    # so we handle this inside the sync wrapper.
+                    # In particular, CO_COROUTINE is different from CO_ASYNC_GENERATOR.
+                    # Flags are listed from LSB here:
+                    # https://docs.python.org/3/library/inspect.html#inspect-module-co-flags
+                    # See also: https://groups.google.com/g/python-tulip/c/6rWweGXLutU?pli=1
+                    return _ahandle_generator(span, ctx_token, res)
 
                 try:
                     if not ignore_output:
@@ -161,6 +170,8 @@ def aentity_method(
 
                 # span will be ended in the generator
                 if isinstance(res, types.AsyncGeneratorType):
+                    # probably unreachable, read the comment in the similar
+                    # part of the sync wrapper.
                     return await _ahandle_generator(span, ctx_token, res)
 
                 try:
@@ -185,24 +196,22 @@ def aentity_method(
     return decorate
 
 
-def _handle_generator(span, res):
-    # for some reason the SPAN_KEY is not being set in the context of the generator, so we re-set it
-    context_api.attach(trace.set_span_in_context(span))
+def _handle_generator(span, ctx_token, res):
     yield from res
 
     span.end()
-
-    # Note: we don't detach the context here as this fails in some situations
-    # https://github.com/open-telemetry/opentelemetry-python/issues/2606
-    # This is not a problem since the context will be detached automatically during garbage collection
+    if ctx_token is not None:
+        context_api.detach(ctx_token)
 
 
 async def _ahandle_generator(span, ctx_token, res):
+    # async with contextlib.aclosing(res) as closing_gen:
     async for part in res:
         yield part
 
     span.end()
-    context_api.detach(ctx_token)
+    if ctx_token is not None:
+        context_api.detach(ctx_token)
 
 
 def _process_exception(span: Span, e: Exception):

lmnr/{openllmetry_sdk → opentelemetry_lib}/opentelemetry/instrumentation/google_genai/utils.py

@@ -34,7 +34,7 @@ def dont_throw(func):
             return func(*args, **kwargs)
         except Exception as e:
             logger.debug(
-                "OpenLLMetry failed to trace in %s, error: %s",
+                "Laminar failed to trace in %s, error: %s",
                 func.__name__,
                 traceback.format_exc(),
             )

lmnr/opentelemetry_lib/tracing/__init__.py

@@ -0,0 +1,139 @@
+import atexit
+import logging
+
+from lmnr.opentelemetry_lib.tracing.processor import LaminarSpanProcessor
+from lmnr.sdk.client.asynchronous.async_client import AsyncLaminarClient
+from lmnr.sdk.client.synchronous.sync_client import LaminarClient
+from lmnr.sdk.log import VerboseColorfulFormatter
+from lmnr.opentelemetry_lib.tracing.instruments import (
+    Instruments,
+    init_instrumentations,
+)
+
+from opentelemetry import trace
+from opentelemetry.sdk.resources import Resource
+from opentelemetry.sdk.trace import TracerProvider, SpanProcessor
+from opentelemetry.sdk.trace.export import SpanExporter
+
+from typing import Optional, Set
+
+
+module_logger = logging.getLogger(__name__)
+console_log_handler = logging.StreamHandler()
+console_log_handler.setFormatter(VerboseColorfulFormatter())
+module_logger.addHandler(console_log_handler)
+
+
+TRACER_NAME = "lmnr.tracer"
+
+MAX_EVENTS_OR_ATTRIBUTES_PER_SPAN = 5000
+
+
+class TracerWrapper(object):
+    resource_attributes: dict = {}
+    enable_content_tracing: bool = True
+    __tracer_provider: Optional[TracerProvider] = None
+    __logger: logging.Logger
+    __client: LaminarClient
+    __async_client: AsyncLaminarClient
+    __resource: Resource
+    __span_processor: SpanProcessor
+
+    def __new__(
+        cls,
+        disable_batch=False,
+        exporter: Optional[SpanExporter] = None,
+        instruments: Optional[Set[Instruments]] = None,
+        block_instruments: Optional[Set[Instruments]] = None,
+        base_url: str = "https://api.lmnr.ai",
+        port: int = 8443,
+        http_port: int = 443,
+        project_api_key: Optional[str] = None,
+        max_export_batch_size: Optional[int] = None,
+        force_http: bool = False,
+        timeout_seconds: int = 10,
+    ) -> "TracerWrapper":
+        base_http_url = f"{base_url}:{http_port}"
+        cls._initialize_logger(cls)
+        if not hasattr(cls, "instance"):
+            obj = cls.instance = super(TracerWrapper, cls).__new__(cls)
+
+            obj.__client = LaminarClient(
+                base_url=base_http_url,
+                project_api_key=project_api_key,
+            )
+            obj.__async_client = AsyncLaminarClient(
+                base_url=base_http_url,
+                project_api_key=project_api_key,
+            )
+
+            obj.__resource = Resource(attributes=TracerWrapper.resource_attributes)
+            obj.__tracer_provider = TracerProvider(resource=obj.__resource)
+
+            obj.__span_processor = LaminarSpanProcessor(
+                base_url=base_url,
+                api_key=project_api_key,
+                port=http_port if force_http else port,
+                exporter=exporter,
+                max_export_batch_size=max_export_batch_size,
+                timeout_seconds=timeout_seconds,
+                force_http=force_http,
+                disable_batch=disable_batch,
+            )
+
+            obj.__tracer_provider.add_span_processor(obj.__span_processor)
+
+            init_instrumentations(
+                tracer_provider=obj.__tracer_provider,
+                instruments=instruments,
+                block_instruments=block_instruments,
+                client=obj.__client,
+                async_client=obj.__async_client,
+            )
+
+            # Force flushes for debug environments (e.g. local development)
+            atexit.register(obj.exit_handler)
+
+        return cls.instance
+
+    def exit_handler(self):
+        if isinstance(self.__span_processor, LaminarSpanProcessor):
+            self.__span_processor.clear()
+        self.flush()
+
+    def _initialize_logger(self):
+        self.__logger = logging.getLogger(__name__)
+        console_log_handler = logging.StreamHandler()
+        console_log_handler.setFormatter(VerboseColorfulFormatter())
+        self.__logger.addHandler(console_log_handler)
+
+    @staticmethod
+    def set_static_params(
+        resource_attributes: dict,
+        enable_content_tracing: bool,
+    ) -> None:
+        TracerWrapper.resource_attributes = resource_attributes
+        TracerWrapper.enable_content_tracing = enable_content_tracing
+
+    @classmethod
+    def verify_initialized(cls) -> bool:
+        return hasattr(cls, "instance")
+
+    @classmethod
+    def clear(cls):
+        # Any state cleanup. Now used in between tests
+        if isinstance(cls.instance.__span_processor, LaminarSpanProcessor):
+            cls.instance.__span_processor.clear()
+
+    def shutdown(self):
+        if self.__tracer_provider is None:
+            return
+        self.__tracer_provider.shutdown()
+
+    def flush(self):
+        return self.__span_processor.force_flush()
+
+    def get_tracer(self):
+        if self.__tracer_provider is None:
+            return trace.get_tracer_provider().get_tracer(TRACER_NAME)
+        return self.__tracer_provider.get_tracer(TRACER_NAME)