lmnr 0.6.8__py3-none-any.whl → 0.6.9__py3-none-any.whl

lmnr/cli.py

@@ -1,6 +1,8 @@
 from argparse import ArgumentParser
 import asyncio
+import glob
 import importlib.util
+import json
 import os
 import re
 import sys
@@ -23,27 +25,27 @@ def add_cursor_rules():
     """Download laminar.mdc file from a hardcoded public URL and save it to .cursor/rules/laminar.mdc"""
     # Hardcoded URL for the laminar.mdc file
     url = "https://raw.githubusercontent.com/lmnr-ai/lmnr/dev/rules/laminar.mdc"
-    
+
     # Create .cursor/rules directory if it doesn't exist
     rules_dir = Path(".cursor/rules")
     rules_dir.mkdir(parents=True, exist_ok=True)
-    
+
     # Define the target file path
     target_file = rules_dir / "laminar.mdc"
-    
+
     try:
         LOG.info(f"Downloading laminar.mdc from {url}")
-        
+
         # Download the file
         with urllib.request.urlopen(url) as response:
             content = response.read()
-        
+
         # Write the content to the target file (this will overwrite if it exists)
-        with open(target_file, 'wb') as f:
+        with open(target_file, "wb") as f:
             f.write(content)
-        
+
         LOG.info(f"Successfully downloaded laminar.mdc to {target_file}")
-        
+
     except urllib.error.URLError as e:
         LOG.error(f"Failed to download file from {url}: {e}")
         sys.exit(1)
@@ -55,7 +57,7 @@ def add_cursor_rules():
 async def run_evaluation(args):
     sys.path.append(os.getcwd())
 
-    if args.file is None:
+    if len(args.file) == 0:
         files = [
             os.path.join(EVAL_DIR, f)
             for f in os.listdir(EVAL_DIR)
@@ -71,9 +73,17 @@ async def run_evaluation(args):
         LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
 
     else:
-        files = [args.file]
+        files = []
+        for pattern in args.file:
+            matches = glob.glob(pattern)
+            if matches:
+                files.extend(matches)
+            else:
+                # If no matches found, treat as literal filename
+                files.append(pattern)
 
     prep_token = PREPARE_ONLY.set(True)
+    scores = []
     try:
         for file in files:
             LOG.info(f"Running evaluation from {file}")
@@ -83,9 +93,9 @@ async def run_evaluation(args):
             spec = importlib.util.spec_from_file_location(name, file)
             if spec is None or spec.loader is None:
                 LOG.error(f"Could not load module specification from {file}")
-                if args.fail_on_error:
-                    return
-                continue
+                if args.continue_on_error:
+                    continue
+                return
             mod = importlib.util.module_from_spec(spec)
             sys.modules[name] = mod
 
@@ -93,19 +103,29 @@ async def run_evaluation(args):
             evaluations: list[Evaluation] | None = EVALUATION_INSTANCES.get()
             if evaluations is None:
                 LOG.warning("Evaluation instance not found")
-                if args.fail_on_error:
-                    return
-                continue
+                if args.continue_on_error:
+                    continue
+                return
 
             LOG.info(f"Loaded {len(evaluations)} evaluations from {file}")
 
             for evaluation in evaluations:
                 try:
-                    await evaluation.run()
+                    eval_scores = await evaluation.run()
+                    scores.append(
+                        {
+                            "file": file,
+                            "scores": eval_scores,
+                        }
+                    )
                 except Exception as e:
                     LOG.error(f"Error running evaluation: {e}")
-                    if args.fail_on_error:
+                    if not args.continue_on_error:
                         raise
+
+        if args.output_file:
+            with open(args.output_file, "w") as f:
+                json.dump(scores, f, indent=2)
     finally:
         PREPARE_ONLY.reset(prep_token)
 
@@ -125,21 +145,27 @@ def cli():
     )
     parser_eval.add_argument(
         "file",
-        nargs="?",
-        help="A file containing the evaluation to run."
+        nargs="*",
+        help="Files or a file containing the evaluation to run."
         + "If no file name is provided, all evaluation files in the `evals` directory are run as long"
         + "as they match *_eval.py or eval_*.py",
-        default=None,
+        default=[],
     )
 
     parser_eval.add_argument(
-        "--fail-on-error",
+        "--continue-on-error",
         action="store_true",
         default=False,
-        help="Fail on error",
+        help="Continue execution upon errors",
+    )
+
+    parser_eval.add_argument(
+        "--output-file",
+        help="Output file to write the results to. Outputs are written in JSON format.",
+        nargs="?",
     )
 
-    parser_download = subparsers.add_parser(
+    subparsers.add_parser(
         "add-cursor-rules",
         description="Download laminar.mdc file and add it to .cursor/rules",
         help="Download laminar.mdc file and add it to .cursor/rules",

lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py

@@ -1,6 +1,7 @@
 """OpenTelemetry Google Generative AI API instrumentation"""
 
 from collections import defaultdict
+import json
 import logging
 import os
 from typing import AsyncGenerator, Callable, Collection, Generator
@@ -11,7 +12,9 @@ from .config import (
     Config,
 )
 from .utils import (
+    ProcessedContentPart,
     dont_throw,
+    get_content,
     role_from_content_union,
     set_span_attribute,
     process_content_union,
@@ -159,7 +162,7 @@ def _set_request_attributes(span, args, kwargs):
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
-                process_content_union(system_instruction),
+                (get_content(process_content_union(system_instruction)) or {}).get("text", ""),
             )
             set_span_attribute(
                 span, f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role", "system"
@@ -169,11 +172,42 @@ def _set_request_attributes(span, args, kwargs):
         if not isinstance(contents, list):
             contents = [contents]
         for content in contents:
+            processed_content = process_content_union(content)
+            content_str = get_content(processed_content)
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
-                process_content_union(content),
+                (
+                    content_str
+                    if isinstance(content_str, str)
+                    else json.dumps(content_str)
+                ),
             )
+            blocks = (
+                processed_content
+                if isinstance(processed_content, list)
+                else [processed_content]
+            )
+            for j, block in enumerate(blocks):
+                block_dict = to_dict(block)
+                if not block_dict.get("function_call"):
+                    continue
+                function_call = to_dict(block_dict.get("function_call", {}))
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.name",
+                    function_call.get("name"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.id",
+                    function_call.get("id"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.arguments",
+                    json.dumps(function_call.get("arguments")),
+                )
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role",
@@ -218,23 +252,64 @@ def _set_response_attributes(span, response: types.GenerateContentResponse):
         )
 
     if should_send_prompts():
-        if len(candidates) > 1:
-            for i, candidate in enumerate(candidates):
+        set_span_attribute(
+            span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.role", "model"
+        )
+        candidates_list = candidates if isinstance(candidates, list) else [candidates]
+        for i, candidate in enumerate(candidates_list):
+            processed_content = process_content_union(candidate.content)
+            if isinstance(processed_content, list):
+                if all(
+                    isinstance(item, dict) and item.get("type") == "text"
+                    for item in processed_content
+                ):
+                    content_str = processed_content[0]["text"]
+                elif all(
+                    isinstance(item, ProcessedContentPart) and item.content
+                    for item in processed_content
+                ):
+                    content_str = processed_content[0].content
+                else:
+                    content_str = get_content(processed_content)
+            else:
+                content_str = get_content(processed_content)
+            set_span_attribute(
+                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.role", "model"
+            )
+            set_span_attribute(
+                span,
+                f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.content",
+                (
+                    content_str
+                    if isinstance(content_str, str)
+                    else json.dumps(content_str)
+                ),
+            )
+            blocks = (
+                processed_content
+                if isinstance(processed_content, list)
+                else [processed_content]
+            )
+            for j, block in enumerate(blocks):
+                block_dict = to_dict(block)
+                if not block_dict.get("function_call"):
+                    continue
+                function_call = to_dict(block_dict.get("function_call", {}))
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.content",
-                    process_content_union(candidate.content),
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.name",
+                    function_call.get("name"),
                 )
                 set_span_attribute(
-                    span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.role", "assistant"
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.id",
+                    function_call.get("id"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.arguments",
+                    json.dumps(function_call.get("arguments")),
                 )
-        else:
-            set_span_attribute(
-                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.content", response.text
-            )
-            set_span_attribute(
-                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.role", "assistant"
-            )
 
 
 @dont_throw
@@ -433,7 +508,7 @@ class GoogleGenAiSdkInstrumentor(BaseInstrumentor):
 
     def _instrument(self, **kwargs):
         tracer_provider = kwargs.get("tracer_provider")
-        tracer = get_tracer(__name__, "0.0.1a0", tracer_provider)
+        tracer = get_tracer(__name__, "0.0.1a1", tracer_provider)
 
         for wrapped_method in WRAPPED_METHODS:
             wrap_function_wrapper(

lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/utils.py

@@ -1,6 +1,5 @@
 import logging
 import traceback
-import json
 
 from .config import (
     Config,
@@ -9,7 +8,28 @@ from google.genai import types
 from google.genai._common import BaseModel
 import pydantic
 from opentelemetry.trace import Span
-from typing import Any
+from typing import Any, Literal
+
+
+class ToolCall(pydantic.BaseModel):
+    name: str | None = pydantic.Field(default=None)
+    id: str | None = pydantic.Field(default=None)
+    arguments: dict[str, Any] = pydantic.Field(default={})
+
+
+class ImageUrlInner(pydantic.BaseModel):
+    url: str = pydantic.Field(default="")
+
+
+class ImageUrl(pydantic.BaseModel):
+    type: Literal["image_url"] = pydantic.Field(default="image_url")
+    image_url: ImageUrlInner = pydantic.Field(default=ImageUrlInner())
+
+
+class ProcessedContentPart(pydantic.BaseModel):
+    content: str | None = pydantic.Field(default=None)
+    function_call: ToolCall | None = pydantic.Field(default=None)
+    image_url: ImageUrl | None = pydantic.Field(default=None)
 
 
 def set_span_attribute(span: Span, name: str, value: str):
@@ -58,36 +78,40 @@ def to_dict(obj: BaseModel | pydantic.BaseModel | dict) -> dict[str, Any]:
         return dict(obj)
 
 
-def process_content_union(
-    content: types.ContentUnion | types.ContentUnionDict,
-    trace_id: str | None = None,
-    span_id: str | None = None,
-    message_index: int = 0,
-) -> str | None:
-    parts = _process_content_union(content, trace_id, span_id, message_index)
-    if parts is None:
-        return None
-    if isinstance(parts, str):
-        return parts
-    elif isinstance(parts, list):
-        if len(parts) == 1 and isinstance(parts[0], str):
-            return parts[0]
-        return json.dumps(
-            [
-                {"type": "text", "text": part} if isinstance(part, str) else part
-                for part in parts
-            ]
-        )
+def get_content(
+    content: (
+        ProcessedContentPart | dict | list[ProcessedContentPart | dict] | str | None
+    ),
+) -> list[Any] | None:
+    if isinstance(content, dict):
+        return content.get("content") or content.get("image_url")
+    if isinstance(content, ProcessedContentPart):
+        if content.content and isinstance(content.content, str):
+            return {
+                "type": "text",
+                "text": content.content,
+            }
+        elif content.image_url:
+            return content.image_url.model_dump()
+        else:
+            return None
+    elif isinstance(content, list):
+        return [get_content(item) or "" for item in content if item is not None]
+    elif isinstance(content, str):
+        return {
+            "type": "text",
+            "text": content,
+        }
     else:
         return None
 
 
-def _process_content_union(
+def process_content_union(
     content: types.ContentUnion | types.ContentUnionDict,
     trace_id: str | None = None,
     span_id: str | None = None,
     message_index: int = 0,
-) -> str | list[str] | None:
+) -> ProcessedContentPart | dict | list[ProcessedContentPart | dict] | None:
     if isinstance(content, types.Content):
         parts = to_dict(content).get("parts", [])
         return [_process_part(part) for part in parts]
@@ -116,9 +140,9 @@ def _process_part_union(
     span_id: str | None = None,
     message_index: int = 0,
     content_index: int = 0,
-) -> str | None:
+) -> ProcessedContentPart | dict | None:
     if isinstance(content, str):
-        return content
+        return ProcessedContentPart(content=content)
     elif isinstance(content, types.File):
         content_dict = to_dict(content)
         name = (
@@ -126,7 +150,7 @@ def _process_part_union(
             or content_dict.get("display_name")
             or content_dict.get("uri")
         )
-        return f"files/{name}"
+        return ProcessedContentPart(content=f"files/{name}")
     elif isinstance(content, (types.Part, dict)):
         return _process_part(content, trace_id, span_id, message_index, content_index)
     else:
@@ -139,11 +163,9 @@ def _process_part(
     span_id: str | None = None,
     message_index: int = 0,
     content_index: int = 0,
-) -> str | None:
+) -> ProcessedContentPart | dict | None:
     part_dict = to_dict(content)
-    if part_dict.get("text") is not None:
-        return part_dict.get("text")
-    elif part_dict.get("inline_data"):
+    if part_dict.get("inline_data"):
         blob = to_dict(part_dict.get("inline_data"))
         if blob.get("mime_type").startswith("image/"):
             return _process_image_item(
@@ -151,7 +173,19 @@ def _process_part(
             )
         else:
             # currently, only images are supported
-            return blob.get("mime_type") or "unknown_media"
+            return ProcessedContentPart(
+                content=blob.get("mime_type") or "unknown_media"
+            )
+    elif part_dict.get("function_call"):
+        return ProcessedContentPart(
+            function_call=ToolCall(
+                name=part_dict.get("function_call").get("name"),
+                id=part_dict.get("function_call").get("id"),
+                arguments=part_dict.get("function_call").get("args", {}),
+            )
+        )
+    elif part_dict.get("text") is not None:
+        return ProcessedContentPart(content=part_dict.get("text"))
     else:
         return None
 
@@ -159,12 +193,17 @@ def _process_part(
 def role_from_content_union(
     content: types.ContentUnion | types.ContentUnionDict,
 ) -> str | None:
+    role = None
     if isinstance(content, types.Content):
-        return to_dict(content).get("role")
+        role = to_dict(content).get("role")
     elif isinstance(content, list) and len(content) > 0:
-        return role_from_content_union(content[0])
+        role = role_from_content_union(content[0])
+    elif isinstance(content, dict):
+        role = content.get("role")
     else:
         return None
+    return role
+    # return "assistant" if role == "model" else role
 
 
 def with_tracer_wrapper(func):
@@ -179,38 +218,22 @@ def with_tracer_wrapper(func):
     return _with_tracer
 
 
-def _run_async(method):
-    import asyncio
-    import threading
-
-    try:
-        loop = asyncio.get_running_loop()
-    except RuntimeError:
-        loop = None
-
-    if loop and loop.is_running():
-        thread = threading.Thread(target=lambda: asyncio.run(method))
-        thread.start()
-        thread.join()
-    else:
-        asyncio.run(method)
-
-
 def _process_image_item(
     blob: dict[str, Any],
     trace_id: str,
     span_id: str,
     message_index: int,
     content_index: int,
-):
+) -> ProcessedContentPart | dict | None:
     # Convert to openai format, so backends can handle it
     return (
-        {
-            "type": "image_url",
-            "image_url": {
-                "url": f"data:image/{blob.get('mime_type').split('/')[1]};base64,{blob.get('data')}",
-            },
-        }
+        ProcessedContentPart(
+            image_url=ImageUrl(
+                image_url=ImageUrlInner(
+                    url=f"data:image/{blob.get('mime_type').split('/')[1]};base64,{blob.get('data')}",
+                )
+            )
+        )
         if Config.convert_image_to_openai_format
         else blob
     )

lmnr/opentelemetry_lib/opentelemetry/instrumentation/langgraph/__init__.py

@@ -0,0 +1,119 @@
+"""OpenTelemetry Langgraph instrumentation"""
+
+import json
+import logging
+from typing import Collection
+
+from .utils import (
+    with_tracer_wrapper,
+)
+
+from langchain_core.runnables.graph import Graph
+from opentelemetry.trace import Tracer
+from wrapt import wrap_function_wrapper
+from opentelemetry.trace import get_tracer, get_current_span
+
+from lmnr.opentelemetry_lib.tracing.context_properties import (
+    update_association_properties,
+)
+
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from opentelemetry.instrumentation.utils import unwrap
+
+
+logger = logging.getLogger(__name__)
+
+_instruments = ("langgraph >= 0.1.0",)
+
+
+@with_tracer_wrapper
+def wrap_pregel_stream(tracer: Tracer, to_wrap, wrapped, instance, args, kwargs):
+    graph: Graph = instance.get_graph()
+    nodes = [
+        {
+            "id": node.id,
+            "name": node.name,
+            "metadata": node.metadata,
+        }
+        for node in graph.nodes.values()
+    ]
+    edges = [
+        {
+            "source": edge.source,
+            "target": edge.target,
+            "conditional": edge.conditional,
+        }
+        for edge in graph.edges
+    ]
+    update_association_properties(
+        {
+            "langgraph.edges": json.dumps(edges),
+            "langgraph.nodes": json.dumps(nodes),
+        },
+    )
+    return wrapped(*args, **kwargs)
+
+
+@with_tracer_wrapper
+async def async_wrap_pregel_stream(
+    tracer: Tracer, to_wrap, wrapped, instance, args, kwargs
+):
+    graph: Graph = await instance.aget_graph()
+    nodes = [
+        {
+            "id": node.id,
+            "name": node.name,
+            "metadata": node.metadata,
+        }
+        for node in graph.nodes.values()
+    ]
+    edges = [
+        {
+            "source": edge.source,
+            "target": edge.target,
+            "conditional": edge.conditional,
+        }
+        for edge in graph.edges
+    ]
+    update_association_properties(
+        {
+            "langgraph.edges": json.dumps(edges),
+            "langgraph.nodes": json.dumps(nodes),
+        },
+    )
+    return await wrapped(*args, **kwargs)
+
+
+class LanggraphInstrumentor(BaseInstrumentor):
+    """An instrumentor for Langgraph."""
+
+    def __init__(self):
+        super().__init__()
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return _instruments
+
+    def _instrument(self, **kwargs):
+        tracer_provider = kwargs.get("tracer_provider")
+        tracer = get_tracer(__name__, "0.0.1a0", tracer_provider)
+
+        wrap_function_wrapper(
+            module="langgraph.pregel",
+            name="Pregel.stream",
+            wrapper=wrap_pregel_stream(tracer, "Pregel.stream"),
+        )
+        wrap_function_wrapper(
+            module="langgraph.pregel",
+            name="Pregel.astream",
+            wrapper=async_wrap_pregel_stream(tracer, "Pregel.astream"),
+        )
+
+    def _uninstrument(self, **kwargs):
+        unwrap(
+            module="langgraph.pregel",
+            name="Pregel.stream",
+        )
+        unwrap(
+            module="langgraph.pregel",
+            name="Pregel.astream",
+        )

lmnr/opentelemetry_lib/opentelemetry/instrumentation/langgraph/utils.py

@@ -0,0 +1,60 @@
+import logging
+import traceback
+
+import pydantic
+from opentelemetry.trace import Span
+from typing import Any
+
+
+def set_span_attribute(span: Span, name: str, value: str):
+    if value is not None:
+        if value != "":
+            span.set_attribute(name, value)
+    return
+
+
+def dont_throw(func):
+    """
+    A decorator that wraps the passed in function and logs exceptions instead of throwing them.
+
+    @param func: The function to wrap
+    @return: The wrapper function
+    """
+    # Obtain a logger specific to the function's module
+    logger = logging.getLogger(func.__module__)
+
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception:
+            logger.debug(
+                "Laminar failed to trace in %s, error: %s",
+                func.__name__,
+                traceback.format_exc(),
+            )
+
+    return wrapper
+
+
+def to_dict(obj: pydantic.BaseModel | dict) -> dict[str, Any]:
+    try:
+        if isinstance(obj, pydantic.BaseModel):
+            return obj.model_dump()
+        elif isinstance(obj, dict):
+            return obj
+        else:
+            return dict(obj)
+    except Exception:
+        return dict(obj)
+
+
+def with_tracer_wrapper(func):
+    """Helper for providing tracer for wrapper functions."""
+
+    def _with_tracer(tracer, to_wrap):
+        def wrapper(wrapped, instance, args, kwargs):
+            return func(tracer, to_wrap, wrapped, instance, args, kwargs)
+
+        return wrapper
+
+    return _with_tracer

lmnr/opentelemetry_lib/tracing/_instrument_initializers.py

@@ -171,6 +171,18 @@ class LangchainInstrumentorInitializer(InstrumentorInitializer):
         return LangchainInstrumentor()
 
 
+class LanggraphInstrumentorInitializer(InstrumentorInitializer):
+    def init_instrumentor(self, *args, **kwargs) -> BaseInstrumentor | None:
+        if not is_package_installed("langgraph"):
+            return None
+        if not is_package_installed("langchain-core"):
+            return None
+
+        from ..opentelemetry.instrumentation.langgraph import LanggraphInstrumentor
+
+        return LanggraphInstrumentor()
+
+
 class LlamaIndexInstrumentorInitializer(InstrumentorInitializer):
     def init_instrumentor(self, *args, **kwargs) -> BaseInstrumentor | None:
         if not (

lmnr/opentelemetry_lib/tracing/context_properties.py

@@ -6,7 +6,7 @@ from lmnr.opentelemetry_lib.tracing.attributes import (
 )
 
 from opentelemetry.context import Context, attach, set_value, get_value
-from opentelemetry.trace import Span
+from opentelemetry.sdk.trace import Span
 from opentelemetry import trace
 
 
@@ -51,8 +51,15 @@ def remove_association_properties(properties: dict) -> None:
 
 
 def _set_association_properties_attributes(span: Span, properties: dict) -> None:
+    if not span.is_recording():
+        return
     for key, value in properties.items():
         if key == TRACING_LEVEL:
             span.set_attribute(f"lmnr.internal.{TRACING_LEVEL}", value)
             continue
+        if (
+            key in ["langgraph.edges", "langgraph.nodes"]
+            and span.name != "LangGraph.workflow"
+        ):
+            continue
         span.set_attribute(f"{ASSOCIATION_PROPERTIES}.{key}", value)

lmnr/opentelemetry_lib/tracing/instruments.py

@@ -26,6 +26,7 @@ class Instruments(Enum):
     HAYSTACK = "haystack"
     LANCEDB = "lancedb"
     LANGCHAIN = "langchain"
+    LANGGRAPH = "langgraph"
     LLAMA_INDEX = "llama_index"
     MARQO = "marqo"
     MCP = "mcp"
@@ -62,6 +63,7 @@ INSTRUMENTATION_INITIALIZERS: dict[
     Instruments.HAYSTACK: initializers.HaystackInstrumentorInitializer(),
     Instruments.LANCEDB: initializers.LanceDBInstrumentorInitializer(),
     Instruments.LANGCHAIN: initializers.LangchainInstrumentorInitializer(),
+    Instruments.LANGGRAPH: initializers.LanggraphInstrumentorInitializer(),
     Instruments.LLAMA_INDEX: initializers.LlamaIndexInstrumentorInitializer(),
     Instruments.MARQO: initializers.MarqoInstrumentorInitializer(),
     Instruments.MCP: initializers.MCPInstrumentorInitializer(),

lmnr/opentelemetry_lib/tracing/processor.py

@@ -6,7 +6,7 @@ from opentelemetry.sdk.trace.export import (
     BatchSpanProcessor,
     SimpleSpanProcessor,
 )
-from opentelemetry.trace import Span
+from opentelemetry.sdk.trace import Span
 from opentelemetry.context import Context, get_value, get_current, set_value
 
 from lmnr.opentelemetry_lib.tracing.attributes import (

lmnr/sdk/evaluations.py

@@ -233,10 +233,10 @@ class Evaluation:
             export_timeout_seconds=trace_export_timeout_seconds,
         )
 
-    async def run(self) -> Awaitable[None]:
+    async def run(self) -> Awaitable[dict[str, int | float]]:
         return await self._run()
 
-    async def _run(self) -> None:
+    async def _run(self) -> dict[str, int | float]:
         if isinstance(self.data, LaminarDataset):
             self.data.set_client(
                 LaminarClient(
@@ -261,11 +261,12 @@ class Evaluation:
         except Exception as e:
             self.reporter.stopWithError(e)
             await self._shutdown()
-            return
+            return {}
 
         average_scores = get_average_scores(result_datapoints)
         self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
         await self._shutdown()
+        return average_scores
 
     async def _shutdown(self):
         # We use flush() instead of shutdown() because multiple evaluations

lmnr/sdk/types.py

@@ -81,7 +81,7 @@ class PartialEvaluationDatapoint(pydantic.BaseModel):
                 "traceId": str(self.trace_id),
                 "executorSpanId": str(self.executor_span_id),
                 "metadata": (
-                    serialize(self.metadata) if self.metadata is not None else None
+                    serialize(self.metadata) if self.metadata is not None else {}
                 ),
             }
         except Exception as e:
@@ -123,7 +123,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
                 "executorSpanId": str(self.executor_span_id),
                 "index": self.index,
                 "metadata": (
-                    serialize(self.metadata) if self.metadata is not None else None
+                    serialize(self.metadata) if self.metadata is not None else {}
                 ),
             }
         except Exception as e:

lmnr/version.py

@@ -3,7 +3,7 @@ import httpx
 from packaging import version
 
 
-__version__ = "0.6.8"
+__version__ = "0.6.9"
 PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
 
 

{lmnr-0.6.8.dist-info → lmnr-0.6.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lmnr
-Version: 0.6.8
+Version: 0.6.9
 Summary: Python SDK for Laminar
 License: Apache-2.0
 Author: lmnr.ai
@@ -46,61 +46,61 @@ Requires-Dist: httpx (>=0.25.0)
 Requires-Dist: opentelemetry-api (>=1.33.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.33.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.33.0)
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.5) ; extra == "alephalpha"
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.5) ; extra == "anthropic"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.5) ; extra == "bedrock"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.5) ; extra == "chromadb"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.5) ; extra == "cohere"
-Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.5) ; extra == "crewai"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.5) ; extra == "google-generativeai"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.5) ; extra == "groq"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.5) ; extra == "haystack"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.5) ; extra == "lancedb"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.5) ; extra == "langchain"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.5) ; extra == "llamaindex"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.5) ; extra == "marqo"
-Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.5) ; extra == "mcp"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.5) ; extra == "milvus"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.5) ; extra == "mistralai"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.5) ; extra == "ollama"
-Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.5) ; extra == "openai"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.5) ; extra == "pinecone"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.5) ; extra == "qdrant"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.5) ; extra == "replicate"
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.5) ; extra == "sagemaker"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.8) ; extra == "alephalpha"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.8) ; extra == "anthropic"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.8) ; extra == "bedrock"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.8) ; extra == "chromadb"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.8) ; extra == "cohere"
+Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.8) ; extra == "crewai"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.8) ; extra == "google-generativeai"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.8) ; extra == "groq"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.8) ; extra == "haystack"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.8) ; extra == "lancedb"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.8) ; extra == "langchain"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.8) ; extra == "llamaindex"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.8) ; extra == "marqo"
+Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.8) ; extra == "mcp"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.8) ; extra == "milvus"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.8) ; extra == "mistralai"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.8) ; extra == "ollama"
+Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.8) ; extra == "openai"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.8) ; extra == "pinecone"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.8) ; extra == "qdrant"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.8) ; extra == "replicate"
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.8) ; extra == "sagemaker"
 Requires-Dist: opentelemetry-instrumentation-threading (>=0.54b0)
-Requires-Dist: opentelemetry-instrumentation-together (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-together (>=0.40.5) ; extra == "together"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.5) ; extra == "transformers"
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.5) ; extra == "vertexai"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.5) ; extra == "watsonx"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.5) ; extra == "weaviate"
+Requires-Dist: opentelemetry-instrumentation-together (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-together (>=0.40.8) ; extra == "together"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.8) ; extra == "transformers"
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.8) ; extra == "vertexai"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.8) ; extra == "watsonx"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.8) ; extra == "weaviate"
 Requires-Dist: opentelemetry-sdk (>=1.33.0)
 Requires-Dist: opentelemetry-semantic-conventions (>=0.54b0)
 Requires-Dist: opentelemetry-semantic-conventions-ai (>=0.4.8)

{lmnr-0.6.8.dist-info → lmnr-0.6.9.dist-info}/RECORD

@@ -1,18 +1,20 @@
 lmnr/__init__.py,sha256=eJ-gIHEk8KV-BeaU8c9spQww_T2G5_OMu4F8JEzngvA,1281
-lmnr/cli.py,sha256=YqemvkUEXMJe_xVXmqX8AyMdnRLvSBLsuoCc6gz7H3g,4834
+lmnr/cli.py,sha256=X5YVOBjitRdZPfcF3qxR4SflgTMZ2wsW6s_yXgvdxkU,5621
 lmnr/opentelemetry_lib/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/opentelemetry_lib/__init__.py,sha256=E_NwAWxh3hckZjXTA80hOzmRUL8RvnSYRdcPExwVROc,2056
 lmnr/opentelemetry_lib/decorators/__init__.py,sha256=45HVoYnHC1Y9D_VSkioDbqD3gm4RPC5sKoztomBI5j8,8496
-lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py,sha256=8bBzOZ4ekeF121KDSmaax9OVUPBrPKXDbq8amwsnq6U,15326
+lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py,sha256=6Fvkc_zZEX1lk8g6ZGFrADLNOL055pkMdO-hEef8qBY,18525
 lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/config.py,sha256=25zevJ7g3MtJP_5gju3jBH7-wg7SbDkktysuUO29ksI,245
-lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/utils.py,sha256=3knAm34hX78JUWvz8rdJSfpytclVnmivATm4q1l1WwU,6144
+lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/utils.py,sha256=ICQENOiICTKodjZVHhq3H5RIRY5bbuWp_KmzkDNgDRM,7471
+lmnr/opentelemetry_lib/opentelemetry/instrumentation/langgraph/__init__.py,sha256=khLC07QIsU15Sv8_Ax0ePjeTg7yP0vwhxvJIAqhU_4A,3099
+lmnr/opentelemetry_lib/opentelemetry/instrumentation/langgraph/utils.py,sha256=nf9sJZXnnts4gYZortEiDvwYjYqYJZTAT0zutuP_R6Y,1512
 lmnr/opentelemetry_lib/tracing/__init__.py,sha256=dy3zrgeiZmLX-aLXuRUW8-RGR4FaIdRRKKEVR0eItGs,5662
-lmnr/opentelemetry_lib/tracing/_instrument_initializers.py,sha256=M5Q1Nbb_oZ6_5f2E-bGB3fTLe_THJnyxoPGroxd8B7s,14145
+lmnr/opentelemetry_lib/tracing/_instrument_initializers.py,sha256=RYSp4PxrF8yfG5qy0ALpx1EArLFBPDLQd6H6YeYw184,14567
 lmnr/opentelemetry_lib/tracing/attributes.py,sha256=MvowVluXfCqSIC3Cvx3tWDqB0Cpr9bpSlY91qL4Iy74,1497
-lmnr/opentelemetry_lib/tracing/context_properties.py,sha256=oiVjotTgSZLoeIo2SeKYIvyR0ZZSHCuj0LwEjjDBrMc,2110
+lmnr/opentelemetry_lib/tracing/context_properties.py,sha256=aWbvMdWB4Q7uqc0GGSsjcRXMTcO18aWOaIZe3QyS_aA,2314
 lmnr/opentelemetry_lib/tracing/exporter.py,sha256=avjmH9hz6PXbMV2bnx7b9GuGC6flAUDwvcxoZMbTWgM,2082
-lmnr/opentelemetry_lib/tracing/instruments.py,sha256=iA8PWSq0gmdgBqtq1GVBynSP64SAri1tkHBojGUnFvA,5163
-lmnr/opentelemetry_lib/tracing/processor.py,sha256=0iGpByt-GT-Z5ddiqSVpymeVIVwOkuABES0tep5Hpvw,3437
+lmnr/opentelemetry_lib/tracing/instruments.py,sha256=dsStEEAFfZ70MHn6WcmRCdDn9Yg-aCvfA6VJs_IvOOs,5267
+lmnr/opentelemetry_lib/tracing/processor.py,sha256=YJ_2j02oU_YZD8cuDltXkCU_fN1hcACI61X5qgCZd7s,3441
 lmnr/opentelemetry_lib/tracing/tracer.py,sha256=oNC6V8eFvuK3i5IWXsKDjEMFL_axeSov3L1fPevwuWM,476
 lmnr/opentelemetry_lib/utils/__init__.py,sha256=pNhf0G3vTd5ccoc03i1MXDbricSaiqCbi1DLWhSekK8,604
 lmnr/opentelemetry_lib/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
@@ -43,14 +45,14 @@ lmnr/sdk/client/synchronous/sync_client.py,sha256=kPS14M0e99xMtLQ_yEOJrFpQWhstqA
 lmnr/sdk/datasets.py,sha256=jl5Wj5nEI9pww4Jwn4XKF8h0gXBU4TOIrhqNjTJsHZQ,1709
 lmnr/sdk/decorators.py,sha256=1uu9xxBYgblFqlhQqH17cZYq7babAmB1lEtvBgTsP0E,4468
 lmnr/sdk/eval_control.py,sha256=KROUrDhcZTrptRZ-hxvr60_o_Gt_8u045jb4cBXcuoY,184
-lmnr/sdk/evaluations.py,sha256=LxeKOVqUYEuyjC05fSknX5HobuIX_xHTn43RGPziLNg,21244
+lmnr/sdk/evaluations.py,sha256=ORGfoyxGJmqBTW3yd1mslr4TK0rWlySZLT4GlC6gVuI,21313
 lmnr/sdk/laminar.py,sha256=Ha6pJSzUqZQBJNfDtEXwYuMcyRvmsZFj7y3A7f0-Y5I,33785
 lmnr/sdk/log.py,sha256=nt_YMmPw1IRbGy0b7q4rTtP4Yo3pQfNxqJPXK3nDSNQ,2213
-lmnr/sdk/types.py,sha256=wnC1OkF-FErpAaRbEGYtiDnPBYw2qxmz3zu8kVBMn7w,12657
+lmnr/sdk/types.py,sha256=5tEX7yoemb9wYyXLy4aqdazudO5I8dglU5A-IegDhsQ,12653
 lmnr/sdk/utils.py,sha256=yrcHIhoADf9lWH9qJWZMmkRWYvd0DuxPSLP3mY6YFw0,4327
-lmnr/version.py,sha256=1Iw9jVY-TZ4TVki4uCzKz9niUMjAWembISjpqLdwenw,1321
-lmnr-0.6.8.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.6.8.dist-info/METADATA,sha256=V9zX-dtUrd4Hseq7AtFeXhHH61tI8HRdSPI1voHx1zs,15131
-lmnr-0.6.8.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-lmnr-0.6.8.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.6.8.dist-info/RECORD,,
+lmnr/version.py,sha256=By46D14Qb3GmEacSJyQ3-_aLJmKB4PmhLldX9eaM1Ic,1321
+lmnr-0.6.9.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.6.9.dist-info/METADATA,sha256=SYj_iCKcXXUwR8WgW6qs72jYvwVQCKnA9EjpzaqFjRQ,15131
+lmnr-0.6.9.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lmnr-0.6.9.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.6.9.dist-info/RECORD,,