lmnr 0.6.8__py3-none-any.whl → 0.6.10__py3-none-any.whl

lmnr/cli.py

@@ -1,6 +1,8 @@
 from argparse import ArgumentParser
 import asyncio
+import glob
 import importlib.util
+import json
 import os
 import re
 import sys
@@ -23,27 +25,27 @@ def add_cursor_rules():
     """Download laminar.mdc file from a hardcoded public URL and save it to .cursor/rules/laminar.mdc"""
     # Hardcoded URL for the laminar.mdc file
     url = "https://raw.githubusercontent.com/lmnr-ai/lmnr/dev/rules/laminar.mdc"
-    
+
     # Create .cursor/rules directory if it doesn't exist
     rules_dir = Path(".cursor/rules")
     rules_dir.mkdir(parents=True, exist_ok=True)
-    
+
     # Define the target file path
     target_file = rules_dir / "laminar.mdc"
-    
+
     try:
         LOG.info(f"Downloading laminar.mdc from {url}")
-        
+
         # Download the file
         with urllib.request.urlopen(url) as response:
             content = response.read()
-        
+
         # Write the content to the target file (this will overwrite if it exists)
-        with open(target_file, 'wb') as f:
+        with open(target_file, "wb") as f:
             f.write(content)
-        
+
         LOG.info(f"Successfully downloaded laminar.mdc to {target_file}")
-        
+
     except urllib.error.URLError as e:
         LOG.error(f"Failed to download file from {url}: {e}")
         sys.exit(1)
@@ -55,7 +57,7 @@ def add_cursor_rules():
 async def run_evaluation(args):
     sys.path.append(os.getcwd())
 
-    if args.file is None:
+    if len(args.file) == 0:
         files = [
             os.path.join(EVAL_DIR, f)
             for f in os.listdir(EVAL_DIR)
@@ -71,9 +73,17 @@ async def run_evaluation(args):
         LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
 
     else:
-        files = [args.file]
+        files = []
+        for pattern in args.file:
+            matches = glob.glob(pattern)
+            if matches:
+                files.extend(matches)
+            else:
+                # If no matches found, treat as literal filename
+                files.append(pattern)
 
     prep_token = PREPARE_ONLY.set(True)
+    scores = []
     try:
         for file in files:
             LOG.info(f"Running evaluation from {file}")
@@ -83,33 +93,56 @@ async def run_evaluation(args):
             spec = importlib.util.spec_from_file_location(name, file)
             if spec is None or spec.loader is None:
                 LOG.error(f"Could not load module specification from {file}")
-                if args.fail_on_error:
-                    return
-                continue
+                if args.continue_on_error:
+                    continue
+                return
             mod = importlib.util.module_from_spec(spec)
             sys.modules[name] = mod
 
             spec.loader.exec_module(mod)
-            evaluations: list[Evaluation] | None = EVALUATION_INSTANCES.get()
-            if evaluations is None:
-                LOG.warning("Evaluation instance not found")
-                if args.fail_on_error:
-                    return
-                continue
+            evaluations = []
+            try:
+                evaluations: list[Evaluation] | None = EVALUATION_INSTANCES.get()
+                if evaluations is None:
+                    raise LookupError()
+            # may be raised by `get()` or manually by us above
+            except LookupError:
+                log_evaluation_instance_not_found()
+                if args.continue_on_error:
+                    continue
+                return
 
             LOG.info(f"Loaded {len(evaluations)} evaluations from {file}")
 
             for evaluation in evaluations:
                 try:
-                    await evaluation.run()
+                    eval_scores = await evaluation.run()
+                    scores.append(
+                        {
+                            "file": file,
+                            "scores": eval_scores,
+                        }
+                    )
                 except Exception as e:
                     LOG.error(f"Error running evaluation: {e}")
-                    if args.fail_on_error:
+                    if not args.continue_on_error:
                         raise
+
+        if args.output_file:
+            with open(args.output_file, "w") as f:
+                json.dump(scores, f, indent=2)
     finally:
         PREPARE_ONLY.reset(prep_token)
 
 
+def log_evaluation_instance_not_found():
+    LOG.warning(
+        "Evaluation instance not found. "
+        "`evaluate` must be called at the top level of the file, "
+        "not inside a function when running evaluations from the CLI."
+    )
+
+
 def cli():
     parser = ArgumentParser(
         prog="lmnr",
@@ -125,21 +158,27 @@ def cli():
     )
     parser_eval.add_argument(
         "file",
-        nargs="?",
-        help="A file containing the evaluation to run."
+        nargs="*",
+        help="Files or a file containing the evaluation to run."
         + "If no file name is provided, all evaluation files in the `evals` directory are run as long"
         + "as they match *_eval.py or eval_*.py",
-        default=None,
+        default=[],
     )
 
     parser_eval.add_argument(
-        "--fail-on-error",
+        "--continue-on-error",
         action="store_true",
         default=False,
-        help="Fail on error",
+        help="Continue execution upon errors",
+    )
+
+    parser_eval.add_argument(
+        "--output-file",
+        help="Output file to write the results to. Outputs are written in JSON format.",
+        nargs="?",
     )
 
-    parser_download = subparsers.add_parser(
+    subparsers.add_parser(
         "add-cursor-rules",
         description="Download laminar.mdc file and add it to .cursor/rules",
         help="Download laminar.mdc file and add it to .cursor/rules",

lmnr/opentelemetry_lib/__init__.py

@@ -54,6 +54,8 @@ class TracerManager:
 
     @staticmethod
     def flush() -> bool:
+        if not hasattr(TracerManager, "_TracerManager__tracer_wrapper"):
+            return False
         return TracerManager.__tracer_wrapper.flush()
 
     @staticmethod

lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py

@@ -1,6 +1,7 @@
 """OpenTelemetry Google Generative AI API instrumentation"""
 
 from collections import defaultdict
+import json
 import logging
 import os
 from typing import AsyncGenerator, Callable, Collection, Generator
@@ -11,7 +12,9 @@ from .config import (
     Config,
 )
 from .utils import (
+    ProcessedContentPart,
     dont_throw,
+    get_content,
     role_from_content_union,
     set_span_attribute,
     process_content_union,
@@ -159,7 +162,7 @@ def _set_request_attributes(span, args, kwargs):
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
-                process_content_union(system_instruction),
+                (get_content(process_content_union(system_instruction)) or {}).get("text", ""),
             )
             set_span_attribute(
                 span, f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role", "system"
@@ -169,11 +172,42 @@ def _set_request_attributes(span, args, kwargs):
         if not isinstance(contents, list):
             contents = [contents]
         for content in contents:
+            processed_content = process_content_union(content)
+            content_str = get_content(processed_content)
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
-                process_content_union(content),
+                (
+                    content_str
+                    if isinstance(content_str, str)
+                    else json.dumps(content_str)
+                ),
             )
+            blocks = (
+                processed_content
+                if isinstance(processed_content, list)
+                else [processed_content]
+            )
+            for j, block in enumerate(blocks):
+                block_dict = to_dict(block)
+                if not block_dict.get("function_call"):
+                    continue
+                function_call = to_dict(block_dict.get("function_call", {}))
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.name",
+                    function_call.get("name"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.id",
+                    function_call.get("id"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.arguments",
+                    json.dumps(function_call.get("arguments")),
+                )
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role",
@@ -218,23 +252,64 @@ def _set_response_attributes(span, response: types.GenerateContentResponse):
         )
 
     if should_send_prompts():
-        if len(candidates) > 1:
-            for i, candidate in enumerate(candidates):
+        set_span_attribute(
+            span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.role", "model"
+        )
+        candidates_list = candidates if isinstance(candidates, list) else [candidates]
+        for i, candidate in enumerate(candidates_list):
+            processed_content = process_content_union(candidate.content)
+            if isinstance(processed_content, list):
+                if all(
+                    isinstance(item, dict) and item.get("type") == "text"
+                    for item in processed_content
+                ):
+                    content_str = processed_content[0]["text"]
+                elif all(
+                    isinstance(item, ProcessedContentPart) and item.content
+                    for item in processed_content
+                ):
+                    content_str = processed_content[0].content
+                else:
+                    content_str = get_content(processed_content)
+            else:
+                content_str = get_content(processed_content)
+            set_span_attribute(
+                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.role", "model"
+            )
+            set_span_attribute(
+                span,
+                f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.content",
+                (
+                    content_str
+                    if isinstance(content_str, str)
+                    else json.dumps(content_str)
+                ),
+            )
+            blocks = (
+                processed_content
+                if isinstance(processed_content, list)
+                else [processed_content]
+            )
+            for j, block in enumerate(blocks):
+                block_dict = to_dict(block)
+                if not block_dict.get("function_call"):
+                    continue
+                function_call = to_dict(block_dict.get("function_call", {}))
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.content",
-                    process_content_union(candidate.content),
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.name",
+                    function_call.get("name"),
                 )
                 set_span_attribute(
-                    span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.role", "assistant"
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.id",
+                    function_call.get("id"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.arguments",
+                    json.dumps(function_call.get("arguments")),
                 )
-        else:
-            set_span_attribute(
-                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.content", response.text
-            )
-            set_span_attribute(
-                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.role", "assistant"
-            )
 
 
 @dont_throw
@@ -433,7 +508,7 @@ class GoogleGenAiSdkInstrumentor(BaseInstrumentor):
 
     def _instrument(self, **kwargs):
         tracer_provider = kwargs.get("tracer_provider")
-        tracer = get_tracer(__name__, "0.0.1a0", tracer_provider)
+        tracer = get_tracer(__name__, "0.0.1a1", tracer_provider)
 
         for wrapped_method in WRAPPED_METHODS:
             wrap_function_wrapper(

lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/utils.py

@@ -1,6 +1,5 @@
 import logging
 import traceback
-import json
 
 from .config import (
     Config,
@@ -9,7 +8,28 @@ from google.genai import types
 from google.genai._common import BaseModel
 import pydantic
 from opentelemetry.trace import Span
-from typing import Any
+from typing import Any, Literal
+
+
+class ToolCall(pydantic.BaseModel):
+    name: str | None = pydantic.Field(default=None)
+    id: str | None = pydantic.Field(default=None)
+    arguments: dict[str, Any] = pydantic.Field(default={})
+
+
+class ImageUrlInner(pydantic.BaseModel):
+    url: str = pydantic.Field(default="")
+
+
+class ImageUrl(pydantic.BaseModel):
+    type: Literal["image_url"] = pydantic.Field(default="image_url")
+    image_url: ImageUrlInner = pydantic.Field(default=ImageUrlInner())
+
+
+class ProcessedContentPart(pydantic.BaseModel):
+    content: str | None = pydantic.Field(default=None)
+    function_call: ToolCall | None = pydantic.Field(default=None)
+    image_url: ImageUrl | None = pydantic.Field(default=None)
 
 
 def set_span_attribute(span: Span, name: str, value: str):
@@ -58,36 +78,40 @@ def to_dict(obj: BaseModel | pydantic.BaseModel | dict) -> dict[str, Any]:
         return dict(obj)
 
 
-def process_content_union(
-    content: types.ContentUnion | types.ContentUnionDict,
-    trace_id: str | None = None,
-    span_id: str | None = None,
-    message_index: int = 0,
-) -> str | None:
-    parts = _process_content_union(content, trace_id, span_id, message_index)
-    if parts is None:
-        return None
-    if isinstance(parts, str):
-        return parts
-    elif isinstance(parts, list):
-        if len(parts) == 1 and isinstance(parts[0], str):
-            return parts[0]
-        return json.dumps(
-            [
-                {"type": "text", "text": part} if isinstance(part, str) else part
-                for part in parts
-            ]
-        )
+def get_content(
+    content: (
+        ProcessedContentPart | dict | list[ProcessedContentPart | dict] | str | None
+    ),
+) -> list[Any] | None:
+    if isinstance(content, dict):
+        return content.get("content") or content.get("image_url")
+    if isinstance(content, ProcessedContentPart):
+        if content.content and isinstance(content.content, str):
+            return {
+                "type": "text",
+                "text": content.content,
+            }
+        elif content.image_url:
+            return content.image_url.model_dump()
+        else:
+            return None
+    elif isinstance(content, list):
+        return [get_content(item) or "" for item in content if item is not None]
+    elif isinstance(content, str):
+        return {
+            "type": "text",
+            "text": content,
+        }
     else:
         return None
 
 
-def _process_content_union(
+def process_content_union(
     content: types.ContentUnion | types.ContentUnionDict,
     trace_id: str | None = None,
     span_id: str | None = None,
     message_index: int = 0,
-) -> str | list[str] | None:
+) -> ProcessedContentPart | dict | list[ProcessedContentPart | dict] | None:
     if isinstance(content, types.Content):
         parts = to_dict(content).get("parts", [])
         return [_process_part(part) for part in parts]
@@ -116,9 +140,9 @@ def _process_part_union(
     span_id: str | None = None,
     message_index: int = 0,
     content_index: int = 0,
-) -> str | None:
+) -> ProcessedContentPart | dict | None:
     if isinstance(content, str):
-        return content
+        return ProcessedContentPart(content=content)
     elif isinstance(content, types.File):
         content_dict = to_dict(content)
         name = (
@@ -126,7 +150,7 @@ def _process_part_union(
             or content_dict.get("display_name")
             or content_dict.get("uri")
         )
-        return f"files/{name}"
+        return ProcessedContentPart(content=f"files/{name}")
     elif isinstance(content, (types.Part, dict)):
         return _process_part(content, trace_id, span_id, message_index, content_index)
     else:
@@ -139,11 +163,9 @@ def _process_part(
     span_id: str | None = None,
     message_index: int = 0,
     content_index: int = 0,
-) -> str | None:
+) -> ProcessedContentPart | dict | None:
     part_dict = to_dict(content)
-    if part_dict.get("text") is not None:
-        return part_dict.get("text")
-    elif part_dict.get("inline_data"):
+    if part_dict.get("inline_data"):
         blob = to_dict(part_dict.get("inline_data"))
         if blob.get("mime_type").startswith("image/"):
             return _process_image_item(
@@ -151,7 +173,19 @@ def _process_part(
             )
         else:
             # currently, only images are supported
-            return blob.get("mime_type") or "unknown_media"
+            return ProcessedContentPart(
+                content=blob.get("mime_type") or "unknown_media"
+            )
+    elif part_dict.get("function_call"):
+        return ProcessedContentPart(
+            function_call=ToolCall(
+                name=part_dict.get("function_call").get("name"),
+                id=part_dict.get("function_call").get("id"),
+                arguments=part_dict.get("function_call").get("args", {}),
+            )
+        )
+    elif part_dict.get("text") is not None:
+        return ProcessedContentPart(content=part_dict.get("text"))
     else:
         return None
 
@@ -159,12 +193,17 @@ def _process_part(
 def role_from_content_union(
     content: types.ContentUnion | types.ContentUnionDict,
 ) -> str | None:
+    role = None
     if isinstance(content, types.Content):
-        return to_dict(content).get("role")
+        role = to_dict(content).get("role")
     elif isinstance(content, list) and len(content) > 0:
-        return role_from_content_union(content[0])
+        role = role_from_content_union(content[0])
+    elif isinstance(content, dict):
+        role = content.get("role")
     else:
         return None
+    return role
+    # return "assistant" if role == "model" else role
 
 
 def with_tracer_wrapper(func):
@@ -179,38 +218,22 @@ def with_tracer_wrapper(func):
     return _with_tracer
 
 
-def _run_async(method):
-    import asyncio
-    import threading
-
-    try:
-        loop = asyncio.get_running_loop()
-    except RuntimeError:
-        loop = None
-
-    if loop and loop.is_running():
-        thread = threading.Thread(target=lambda: asyncio.run(method))
-        thread.start()
-        thread.join()
-    else:
-        asyncio.run(method)
-
-
 def _process_image_item(
     blob: dict[str, Any],
     trace_id: str,
     span_id: str,
     message_index: int,
     content_index: int,
-):
+) -> ProcessedContentPart | dict | None:
     # Convert to openai format, so backends can handle it
     return (
-        {
-            "type": "image_url",
-            "image_url": {
-                "url": f"data:image/{blob.get('mime_type').split('/')[1]};base64,{blob.get('data')}",
-            },
-        }
+        ProcessedContentPart(
+            image_url=ImageUrl(
+                image_url=ImageUrlInner(
+                    url=f"data:image/{blob.get('mime_type').split('/')[1]};base64,{blob.get('data')}",
+                )
+            )
+        )
         if Config.convert_image_to_openai_format
         else blob
     )

lmnr/opentelemetry_lib/opentelemetry/instrumentation/langgraph/__init__.py

@@ -0,0 +1,121 @@
+"""OpenTelemetry Langgraph instrumentation"""
+
+import json
+import logging
+from typing import Collection
+
+from .utils import (
+    with_tracer_wrapper,
+)
+
+from langchain_core.runnables.graph import Graph
+from opentelemetry.trace import Tracer
+from wrapt import wrap_function_wrapper
+from opentelemetry.trace import get_tracer
+
+from lmnr.opentelemetry_lib.tracing.context_properties import (
+    update_association_properties,
+)
+
+from opentelemetry.instrumentation.instrumentor import BaseInstrumentor
+from opentelemetry.instrumentation.utils import unwrap
+
+
+logger = logging.getLogger(__name__)
+
+_instruments = ("langgraph >= 0.1.0",)
+
+
+@with_tracer_wrapper
+def wrap_pregel_stream(tracer: Tracer, to_wrap, wrapped, instance, args, kwargs):
+    graph: Graph = instance.get_graph()
+    nodes = [
+        {
+            "id": node.id,
+            "name": node.name,
+            "metadata": node.metadata,
+        }
+        for node in graph.nodes.values()
+    ]
+    edges = [
+        {
+            "source": edge.source,
+            "target": edge.target,
+            "conditional": edge.conditional,
+        }
+        for edge in graph.edges
+    ]
+    update_association_properties(
+        {
+            "langgraph.edges": json.dumps(edges),
+            "langgraph.nodes": json.dumps(nodes),
+        },
+    )
+    return wrapped(*args, **kwargs)
+
+
+@with_tracer_wrapper
+async def async_wrap_pregel_stream(
+    tracer: Tracer, to_wrap, wrapped, instance, args, kwargs
+):
+    graph: Graph = await instance.aget_graph()
+    nodes = [
+        {
+            "id": node.id,
+            "name": node.name,
+            "metadata": node.metadata,
+        }
+        for node in graph.nodes.values()
+    ]
+    edges = [
+        {
+            "source": edge.source,
+            "target": edge.target,
+            "conditional": edge.conditional,
+        }
+        for edge in graph.edges
+    ]
+    update_association_properties(
+        {
+            "langgraph.edges": json.dumps(edges),
+            "langgraph.nodes": json.dumps(nodes),
+        },
+    )
+
+    async for item in wrapped(*args, **kwargs):
+        yield item
+
+
+class LanggraphInstrumentor(BaseInstrumentor):
+    """An instrumentor for Langgraph."""
+
+    def __init__(self):
+        super().__init__()
+
+    def instrumentation_dependencies(self) -> Collection[str]:
+        return _instruments
+
+    def _instrument(self, **kwargs):
+        tracer_provider = kwargs.get("tracer_provider")
+        tracer = get_tracer(__name__, "0.0.1a0", tracer_provider)
+
+        wrap_function_wrapper(
+            module="langgraph.pregel",
+            name="Pregel.stream",
+            wrapper=wrap_pregel_stream(tracer, "Pregel.stream"),
+        )
+        wrap_function_wrapper(
+            module="langgraph.pregel",
+            name="Pregel.astream",
+            wrapper=async_wrap_pregel_stream(tracer, "Pregel.astream"),
+        )
+
+    def _uninstrument(self, **kwargs):
+        unwrap(
+            module="langgraph.pregel",
+            name="Pregel.stream",
+        )
+        unwrap(
+            module="langgraph.pregel",
+            name="Pregel.astream",
+        )