lmnr 0.6.11__py3-none-any.whl → 0.6.13__py3-none-any.whl

lmnr/__init__.py

@@ -13,6 +13,7 @@ from .sdk.types import (
 )
 from .sdk.decorators import observe
 from .sdk.types import LaminarSpanContext
+from .opentelemetry_lib.litellm import LaminarLiteLLMCallback
 from .opentelemetry_lib.tracing.attributes import Attributes
 from .opentelemetry_lib.tracing.instruments import Instruments
 from .opentelemetry_lib.tracing.processor import LaminarSpanProcessor
@@ -30,6 +31,7 @@ __all__ = [
     "Laminar",
     "LaminarClient",
     "LaminarDataset",
+    "LaminarLiteLLMCallback",
     "LaminarSpanContext",
     "LaminarSpanProcessor",
     "RunAgentResponseChunk",

lmnr/opentelemetry_lib/__init__.py

@@ -7,7 +7,7 @@ from opentelemetry.sdk.resources import SERVICE_NAME
 from lmnr.opentelemetry_lib.tracing.instruments import Instruments
 from lmnr.opentelemetry_lib.tracing import TracerWrapper
 
-MAX_MANUAL_SPAN_PAYLOAD_SIZE = 1024 * 1024  # 1MB
+MAX_MANUAL_SPAN_PAYLOAD_SIZE = 1024 * 1024 * 10  # 10MB
 
 
 class TracerManager:

lmnr/opentelemetry_lib/litellm/__init__.py

@@ -0,0 +1,371 @@
+"""LiteLLM callback logger for Laminar"""
+
+import json
+from datetime import datetime
+
+from opentelemetry.trace import SpanKind, Status, StatusCode, Tracer
+from lmnr.opentelemetry_lib.litellm.utils import model_as_dict, set_span_attribute
+from lmnr.opentelemetry_lib.tracing import TracerWrapper
+
+from lmnr.opentelemetry_lib.utils.package_check import is_package_installed
+from lmnr.sdk.log import get_default_logger
+
+logger = get_default_logger(__name__)
+
+SUPPORTED_CALL_TYPES = ["completion", "acompletion"]
+
+# Try to import the necessary LiteLLM components and gracefully handle ImportError
+try:
+    if not is_package_installed("litellm"):
+        raise ImportError("LiteLLM is not installed")
+
+    from litellm.integrations.custom_batch_logger import CustomBatchLogger
+
+    class LaminarLiteLLMCallback(CustomBatchLogger):
+        """Custom LiteLLM logger that sends logs to Laminar via OpenTelemetry spans
+
+        Usage:
+            import litellm
+            from lmnr import Laminar, LaminarLiteLLMCallback
+
+            # make sure this comes first
+            Laminar.initialize()
+
+            # Add the logger to LiteLLM callbacks
+            litellm.callbacks = [LaminarLiteLLMCallback()]
+        """
+
+        def __init__(self, **kwargs):
+            super().__init__(**kwargs)
+            if not hasattr(TracerWrapper, "instance") or TracerWrapper.instance is None:
+                raise ValueError("Laminar must be initialized before LiteLLM callback")
+
+        def _get_tracer(self) -> Tracer:
+            if not hasattr(TracerWrapper, "instance") or TracerWrapper.instance is None:
+                raise ValueError("Laminar must be initialized before LiteLLM callback")
+            return TracerWrapper().get_tracer()
+
+        def log_success_event(
+            self, kwargs, response_obj, start_time: datetime, end_time: datetime
+        ):
+            if kwargs.get("call_type") not in SUPPORTED_CALL_TYPES:
+                return
+            try:
+                self._create_span(
+                    kwargs, response_obj, start_time, end_time, is_success=True
+                )
+            except Exception as e:
+                logger.error(f"Error in log_success_event: {e}")
+
+        def log_failure_event(
+            self, kwargs, response_obj, start_time: datetime, end_time: datetime
+        ):
+            if kwargs.get("call_type") not in SUPPORTED_CALL_TYPES:
+                return
+            try:
+                self._create_span(
+                    kwargs, response_obj, start_time, end_time, is_success=False
+                )
+            except Exception as e:
+                logger.error(f"Error in log_failure_event: {e}")
+
+        async def async_log_success_event(
+            self, kwargs, response_obj, start_time: datetime, end_time: datetime
+        ):
+            self.log_success_event(kwargs, response_obj, start_time, end_time)
+
+        async def async_log_failure_event(
+            self, kwargs, response_obj, start_time: datetime, end_time: datetime
+        ):
+            self.log_failure_event(kwargs, response_obj, start_time, end_time)
+
+        def _create_span(
+            self,
+            kwargs,
+            response_obj,
+            start_time: datetime,
+            end_time: datetime,
+            is_success: bool,
+        ):
+            """Create an OpenTelemetry span for the LiteLLM call"""
+            span_name = "litellm.completion"
+            try:
+                tracer = self._get_tracer()
+            except Exception as e:
+                logger.error(f"Error getting tracer: {e}")
+                return
+            span = tracer.start_span(
+                span_name,
+                kind=SpanKind.CLIENT,
+                start_time=int(start_time.timestamp() * 1e9),
+                attributes={
+                    "lmnr.internal.provider": "litellm",
+                },
+            )
+            try:
+                model = kwargs.get("model", "unknown")
+                if kwargs.get("custom_llm_provider"):
+                    set_span_attribute(
+                        span, "gen_ai.system", kwargs["custom_llm_provider"]
+                    )
+
+                messages = kwargs.get("messages", [])
+                self._process_input_messages(span, messages)
+
+                tools = kwargs.get("tools", [])
+                self._process_request_tool_definitions(span, tools)
+
+                set_span_attribute(span, "gen_ai.request.model", model)
+
+                # Add more attributes from kwargs
+                if "temperature" in kwargs:
+                    set_span_attribute(
+                        span, "gen_ai.request.temperature", kwargs["temperature"]
+                    )
+                if "max_tokens" in kwargs:
+                    set_span_attribute(
+                        span, "gen_ai.request.max_tokens", kwargs["max_tokens"]
+                    )
+                if "top_p" in kwargs:
+                    set_span_attribute(span, "gen_ai.request.top_p", kwargs["top_p"])
+
+                if is_success:
+                    span.set_status(Status(StatusCode.OK))
+                    if kwargs.get("complete_streaming_response"):
+                        self._process_success_response(
+                            span,
+                            kwargs.get("complete_streaming_response"),
+                        )
+                    else:
+                        self._process_success_response(span, response_obj)
+                else:
+                    span.set_status(Status(StatusCode.ERROR))
+                    if isinstance(response_obj, Exception):
+                        span.record_exception(response_obj)
+
+            except Exception as e:
+                span.record_exception(e)
+                logger.error(f"Error in Laminar LiteLLM instrumentation: {e}")
+            finally:
+                span.end(int(end_time.timestamp() * 1e9))
+
+        def _process_input_messages(self, span, messages):
+            """Process and set message attributes on the span"""
+            if not isinstance(messages, list):
+                return
+
+            for i, message in enumerate(messages):
+                message_dict = model_as_dict(message)
+                role = message_dict.get("role", "unknown")
+                set_span_attribute(span, f"gen_ai.prompt.{i}.role", role)
+
+                tool_calls = message_dict.get("tool_calls", [])
+                self._process_tool_calls(span, tool_calls, i, is_response=False)
+
+                content = message_dict.get("content", "")
+                if content is None:
+                    continue
+                if isinstance(content, str):
+                    set_span_attribute(span, f"gen_ai.prompt.{i}.content", content)
+                elif isinstance(content, list):
+                    set_span_attribute(
+                        span, f"gen_ai.prompt.{i}.content", json.dumps(content)
+                    )
+                else:
+                    set_span_attribute(
+                        span,
+                        f"gen_ai.prompt.{i}.content",
+                        json.dumps(model_as_dict(content)),
+                    )
+                if role == "tool":
+                    set_span_attribute(
+                        span,
+                        f"gen_ai.prompt.{i}.tool_call_id",
+                        message_dict.get("tool_call_id"),
+                    )
+
+        def _process_request_tool_definitions(self, span, tools):
+            """Process and set tool definitions attributes on the span"""
+            if not isinstance(tools, list):
+                return
+
+            for i, tool in enumerate(tools):
+                tool_dict = model_as_dict(tool)
+                if tool_dict.get("type") != "function":
+                    # TODO: parse other tool types
+                    continue
+
+                function_dict = tool_dict.get("function", {})
+                function_name = function_dict.get("name", "")
+                function_description = function_dict.get("description", "")
+                function_parameters = function_dict.get("parameters", {})
+                set_span_attribute(
+                    span,
+                    f"llm.request.functions.{i}.name",
+                    function_name,
+                )
+                set_span_attribute(
+                    span,
+                    f"llm.request.functions.{i}.description",
+                    function_description,
+                )
+                set_span_attribute(
+                    span,
+                    f"llm.request.functions.{i}.parameters",
+                    json.dumps(function_parameters),
+                )
+
+        def _process_response_usage(self, span, usage):
+            """Process and set usage attributes on the span"""
+            usage_dict = model_as_dict(usage)
+            if (
+                not usage_dict.get("prompt_tokens")
+                and not usage_dict.get("completion_tokens")
+                and not usage_dict.get("total_tokens")
+            ):
+                return
+
+            set_span_attribute(
+                span, "gen_ai.usage.input_tokens", usage_dict.get("prompt_tokens")
+            )
+            set_span_attribute(
+                span, "gen_ai.usage.output_tokens", usage_dict.get("completion_tokens")
+            )
+            set_span_attribute(
+                span, "llm.usage.total_tokens", usage_dict.get("total_tokens")
+            )
+
+            if usage_dict.get("prompt_tokens_details"):
+                details = usage_dict.get("prompt_tokens_details", {})
+                details = model_as_dict(details)
+                if details.get("cached_tokens"):
+                    set_span_attribute(
+                        span,
+                        "gen_ai.usage.cache_read_input_tokens",
+                        details.get("cached_tokens"),
+                    )
+                # TODO: add audio/image/text token details
+            # TODO: add completion tokens details (reasoning tokens)
+
+        def _process_tool_calls(self, span, tool_calls, choice_index, is_response=True):
+            """Process and set tool call attributes on the span"""
+            attr_prefix = "completion" if is_response else "prompt"
+            if not isinstance(tool_calls, list):
+                return
+
+            for j, tool_call in enumerate(tool_calls):
+                tool_call_dict = model_as_dict(tool_call)
+
+                tool_name = tool_call_dict.get(
+                    "name", tool_call_dict.get("function", {}).get("name", "")
+                )
+                set_span_attribute(
+                    span,
+                    f"gen_ai.{attr_prefix}.{choice_index}.tool_calls.{j}.name",
+                    tool_name,
+                )
+
+                call_id = tool_call_dict.get("id", "")
+                set_span_attribute(
+                    span,
+                    f"gen_ai.{attr_prefix}.{choice_index}.tool_calls.{j}.id",
+                    call_id,
+                )
+
+                tool_arguments = tool_call_dict.get(
+                    "arguments", tool_call_dict.get("function", {}).get("arguments", "")
+                )
+                if isinstance(tool_arguments, str):
+                    set_span_attribute(
+                        span,
+                        f"gen_ai.{attr_prefix}.{choice_index}.tool_calls.{j}.arguments",
+                        tool_arguments,
+                    )
+                else:
+                    set_span_attribute(
+                        span,
+                        f"gen_ai.{attr_prefix}.{choice_index}.tool_calls.{j}.arguments",
+                        json.dumps(model_as_dict(tool_arguments)),
+                    )
+
+        def _process_response_choices(self, span, choices):
+            """Process and set choice attributes on the span"""
+            if not isinstance(choices, list):
+                return
+
+            for i, choice in enumerate(choices):
+                choice_dict = model_as_dict(choice)
+                message = choice_dict.get("message", choice_dict)
+
+                role = message.get("role", "unknown")
+                set_span_attribute(span, f"gen_ai.completion.{i}.role", role)
+
+                tool_calls = message.get("tool_calls", [])
+                self._process_tool_calls(span, tool_calls, i, is_response=True)
+
+                content = message.get("content", "")
+                if content is None:
+                    continue
+                if isinstance(content, str):
+                    set_span_attribute(span, f"gen_ai.completion.{i}.content", content)
+                elif isinstance(content, list):
+                    set_span_attribute(
+                        span, f"gen_ai.completion.{i}.content", json.dumps(content)
+                    )
+                else:
+                    set_span_attribute(
+                        span,
+                        f"gen_ai.completion.{i}.content",
+                        json.dumps(model_as_dict(content)),
+                    )
+
+        def _process_success_response(self, span, response_obj):
+            """Process successful response attributes"""
+            response_dict = model_as_dict(response_obj)
+            set_span_attribute(span, "gen_ai.response.id", response_dict.get("id"))
+            set_span_attribute(
+                span, "gen_ai.response.model", response_dict.get("model")
+            )
+
+            if response_dict.get("usage"):
+                self._process_response_usage(span, response_dict.get("usage"))
+
+            if response_dict.get("cache_creation_input_tokens"):
+                set_span_attribute(
+                    span,
+                    "gen_ai.usage.cache_creation_input_tokens",
+                    response_dict.get("cache_creation_input_tokens"),
+                )
+            if response_dict.get("cache_read_input_tokens"):
+                set_span_attribute(
+                    span,
+                    "gen_ai.usage.cache_read_input_tokens",
+                    response_dict.get("cache_read_input_tokens"),
+                )
+
+            if response_dict.get("choices"):
+                self._process_response_choices(span, response_dict.get("choices"))
+
+except ImportError as e:
+    logger.debug(f"LiteLLM callback unavailable: {e}")
+
+    # Create a no-op logger when LiteLLM is not available
+    class LaminarLiteLLMCallback:
+        """No-op logger when LiteLLM is not available"""
+
+        def __init__(self, **kwargs):
+            logger.warning(
+                "LiteLLM is not installed. Install with: pip install litellm"
+            )
+
+        def log_success_event(self, *args, **kwargs):
+            pass
+
+        def log_failure_event(self, *args, **kwargs):
+            pass
+
+        async def async_log_success_event(self, *args, **kwargs):
+            pass
+
+        async def async_log_failure_event(self, *args, **kwargs):
+            pass

lmnr/opentelemetry_lib/litellm/utils.py

@@ -0,0 +1,18 @@
+from pydantic import BaseModel
+from opentelemetry.sdk.trace import Span
+from opentelemetry.util.types import AttributeValue
+
+
+def model_as_dict(model: BaseModel | dict) -> dict:
+    if isinstance(model, BaseModel) and hasattr(model, "model_dump"):
+        return model.model_dump()
+    elif isinstance(model, dict):
+        return model
+    else:
+        return dict(model)
+
+
+def set_span_attribute(span: Span, key: str, value: AttributeValue | None):
+    if value is None or value == "":
+        return
+    span.set_attribute(key, value)

lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py

@@ -12,7 +12,6 @@ from .config import (
     Config,
 )
 from .utils import (
-    ProcessedContentPart,
     dont_throw,
     get_content,
     role_from_content_union,
@@ -130,27 +129,30 @@ def _set_request_attributes(span, args, kwargs):
     )
 
     tools: list[types.FunctionDeclaration] = []
-    if kwargs.get("tools"):
-        for tool in kwargs.get("tools"):
+    arg_tools = config_dict.get("tools", kwargs.get("tools"))
+    if arg_tools:
+        for tool in arg_tools:
             if isinstance(tool, types.Tool):
                 tools += tool.function_declarations or []
             elif isinstance(tool, Callable):
                 tools.append(types.FunctionDeclaration.from_callable(tool))
+
     for tool_num, tool in enumerate(tools):
+        tool_dict = to_dict(tool)
         set_span_attribute(
             span,
             f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{tool_num}.name",
-            to_dict(tool).get("name"),
+            tool_dict.get("name"),
         )
         set_span_attribute(
             span,
             f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{tool_num}.description",
-            to_dict(tool).get("description"),
+            tool_dict.get("description"),
         )
         set_span_attribute(
             span,
             f"{SpanAttributes.LLM_REQUEST_FUNCTIONS}.{tool_num}.parameters",
-            to_dict(tool).get("parameters"),
+            json.dumps(tool_dict.get("parameters")),
         )
 
     if should_send_prompts():
@@ -162,7 +164,9 @@ def _set_request_attributes(span, args, kwargs):
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
-                (get_content(process_content_union(system_instruction)) or {}).get("text", ""),
+                (get_content(process_content_union(system_instruction)) or {}).get(
+                    "text", ""
+                ),
             )
             set_span_attribute(
                 span, f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role", "system"
@@ -174,6 +178,7 @@ def _set_request_attributes(span, args, kwargs):
         for content in contents:
             processed_content = process_content_union(content)
             content_str = get_content(processed_content)
+
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
@@ -188,26 +193,35 @@ def _set_request_attributes(span, args, kwargs):
                 if isinstance(processed_content, list)
                 else [processed_content]
             )
-            for j, block in enumerate(blocks):
+            tool_call_index = 0
+            for block in blocks:
                 block_dict = to_dict(block)
+
                 if not block_dict.get("function_call"):
                     continue
                 function_call = to_dict(block_dict.get("function_call", {}))
+
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.name",
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{tool_call_index}.name",
                     function_call.get("name"),
                 )
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.id",
-                    function_call.get("id"),
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{tool_call_index}.id",
+                    (
+                        function_call.get("id")
+                        if function_call.get("id") is not None
+                        else function_call.get("name")
+                    ),  # google genai doesn't support tool call ids
                 )
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.arguments",
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{tool_call_index}.arguments",
                     json.dumps(function_call.get("arguments")),
                 )
+                tool_call_index += 1
+
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role",
@@ -258,21 +272,8 @@ def _set_response_attributes(span, response: types.GenerateContentResponse):
         candidates_list = candidates if isinstance(candidates, list) else [candidates]
         for i, candidate in enumerate(candidates_list):
             processed_content = process_content_union(candidate.content)
-            if isinstance(processed_content, list):
-                if all(
-                    isinstance(item, dict) and item.get("type") == "text"
-                    for item in processed_content
-                ):
-                    content_str = processed_content[0]["text"]
-                elif all(
-                    isinstance(item, ProcessedContentPart) and item.content
-                    for item in processed_content
-                ):
-                    content_str = processed_content[0].content
-                else:
-                    content_str = get_content(processed_content)
-            else:
-                content_str = get_content(processed_content)
+            content_str = get_content(processed_content)
+
             set_span_attribute(
                 span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.role", "model"
             )
@@ -290,26 +291,33 @@ def _set_response_attributes(span, response: types.GenerateContentResponse):
                 if isinstance(processed_content, list)
                 else [processed_content]
             )
-            for j, block in enumerate(blocks):
+
+            tool_call_index = 0
+            for block in blocks:
                 block_dict = to_dict(block)
                 if not block_dict.get("function_call"):
                     continue
                 function_call = to_dict(block_dict.get("function_call", {}))
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.name",
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{tool_call_index}.name",
                     function_call.get("name"),
                 )
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.id",
-                    function_call.get("id"),
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{tool_call_index}.id",
+                    (
+                        function_call.get("id")
+                        if function_call.get("id") is not None
+                        else function_call.get("name")
+                    ),  # google genai doesn't support tool call ids
                 )
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.arguments",
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{tool_call_index}.arguments",
                     json.dumps(function_call.get("arguments")),
                 )
+                tool_call_index += 1
 
 
 @dont_throw

lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/utils.py

@@ -1,3 +1,4 @@
+import base64
 import logging
 import traceback
 
@@ -74,7 +75,8 @@ def to_dict(obj: BaseModel | pydantic.BaseModel | dict) -> dict[str, Any]:
             return obj
         else:
             return dict(obj)
-    except Exception:
+    except Exception as e:
+        logging.error(f"Error converting to dict: {obj}, error: {e}")
         return dict(obj)
 
 
@@ -96,7 +98,7 @@ def get_content(
         else:
             return None
     elif isinstance(content, list):
-        return [get_content(item) or "" for item in content if item is not None]
+        return [get_content(item) for item in content]
     elif isinstance(content, str):
         return {
             "type": "text",
@@ -226,11 +228,15 @@ def _process_image_item(
     content_index: int,
 ) -> ProcessedContentPart | dict | None:
     # Convert to openai format, so backends can handle it
+    data = blob.get("data")
+    encoded_data = (
+        base64.b64encode(data).decode("utf-8") if isinstance(data, bytes) else data
+    )
     return (
         ProcessedContentPart(
             image_url=ImageUrl(
                 image_url=ImageUrlInner(
-                    url=f"data:image/{blob.get('mime_type').split('/')[1]};base64,{blob.get('data')}",
+                    url=f"data:image/{blob.get('mime_type').split('/')[1]};base64,{encoded_data}",
                 )
             )
         )

lmnr/sdk/evaluations.py

@@ -56,7 +56,11 @@ def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Nu
 
     average_scores = {}
     for key, values in per_score_values.items():
-        average_scores[key] = sum(values) / len(values)
+        scores = [v for v in values if v is not None]
+        
+        # If there are no scores, we don't want to include the key in the average scores
+        if len(scores) > 0:
+            average_scores[key] = sum(scores) / len(scores)
 
     return average_scores
 
@@ -97,8 +101,7 @@ class Evaluation:
         self,
         data: EvaluationDataset | list[Datapoint | dict],
         executor: Any,
-        evaluators: dict[str, EvaluatorFunction],
-        human_evaluators: list[HumanEvaluator] = [],
+        evaluators: dict[str, EvaluatorFunction | HumanEvaluator],
         name: str | None = None,
         group_name: str | None = None,
         concurrency_limit: int = DEFAULT_BATCH_SIZE,
@@ -123,17 +126,15 @@ class Evaluation:
             executor (Callable[..., Any]): The executor function.\
                     Takes the data point + any additional arguments and returns\
                     the output to evaluate.
-            evaluators (dict[str, Callable[..., Any]]): Evaluator functions and\
-                names. Each evaluator function takes the output of the executor\
-                _and_ the target data, and returns a score. The score can be a\
-                single number or a dict of string keys and number values.\
-                If the score is a single number, it will be named after the\
-                evaluator function. Evaluator function names must contain only\
-                letters, digits, hyphens, underscores, or spaces.
-            human_evaluators (list[HumanEvaluator], optional):\
-                [Beta] List of instances of HumanEvaluator. For now, human\
-                evaluator only holds the queue name.
-                Defaults to an empty list.
+            evaluators (dict[str, Callable[..., Any] | HumanEvaluator]): Evaluator\
+                functions and HumanEvaluator instances with names. Each evaluator\
+                function takes the output of the executor _and_ the target data,\
+                and returns a score. The score can be a single number or a dict\
+                of string keys and number values. If the score is a single number,\
+                it will be named after the evaluator function.\
+                HumanEvaluator instances create empty spans for manual evaluation.\
+                Evaluator names must contain only letters, digits, hyphens,\
+                underscores, or spaces.
             name (str | None, optional): Optional name of the evaluation.\
                 Used to identify the evaluation in the group.\
                 If not provided, a random name will be generated.
@@ -194,7 +195,6 @@ class Evaluation:
         self.concurrency_limit = concurrency_limit
         self.batch_size = concurrency_limit
         self._logger = get_default_logger(self.__class__.__name__)
-        self.human_evaluators = human_evaluators
         self.upload_tasks = []
         self.base_http_url = f"{base_url}:{http_port or 443}"
 
@@ -345,24 +345,40 @@ class Evaluation:
             # Iterate over evaluators
             scores: dict[str, Numeric] = {}
             for evaluator_name, evaluator in self.evaluators.items():
-                with L.start_as_current_span(
-                    evaluator_name, input={"output": output, "target": target}
-                ) as evaluator_span:
-                    evaluator_span.set_attribute(SPAN_TYPE, SpanType.EVALUATOR.value)
-                    if is_async(evaluator):
-                        value = await evaluator(output, target)
-                    else:
-                        loop = asyncio.get_event_loop()
-                        value = await loop.run_in_executor(
-                            None, evaluator, output, target
-                        )
-                    L.set_span_output(value)
-
-                # If evaluator returns a single number, use evaluator name as key
-                if isinstance(value, NumericTypes):
-                    scores[evaluator_name] = value
+                # Check if evaluator is a HumanEvaluator instance
+                if isinstance(evaluator, HumanEvaluator):
+                    # Create an empty span for human evaluators
+                    with L.start_as_current_span(
+                        evaluator_name,
+                        input={"output": output, "target": target}
+                    ) as human_evaluator_span:
+                        human_evaluator_span.set_attribute(SPAN_TYPE, SpanType.HUMAN_EVALUATOR.value)
+                        # Human evaluators don't execute automatically, just create the span
+                        L.set_span_output(None)
+                    
+                    # We don't want to save the score for human evaluators
+                    scores[evaluator_name] = None
                 else:
-                    scores.update(value)
+                    # Regular evaluator function
+                    with L.start_as_current_span(
+                        evaluator_name,
+                        input={"output": output, "target": target}
+                    ) as evaluator_span:
+                        evaluator_span.set_attribute(SPAN_TYPE, SpanType.EVALUATOR.value)
+                        if is_async(evaluator):
+                            value = await evaluator(output, target)
+                        else:
+                            loop = asyncio.get_event_loop()
+                            value = await loop.run_in_executor(
+                                None, evaluator, output, target
+                            )
+                        L.set_span_output(value)
+
+                    # If evaluator returns a single number, use evaluator name as key
+                    if isinstance(value, NumericTypes):
+                        scores[evaluator_name] = value
+                    else:
+                        scores.update(value)
 
             trace_id = uuid.UUID(int=evaluation_span.get_span_context().trace_id)
 
@@ -373,10 +389,6 @@ class Evaluation:
             executor_output=output,
             scores=scores,
             trace_id=trace_id,
-            # For now add all human evaluators to all result datapoints
-            # In the future, we will add ways to specify which human evaluators
-            # to add to which result datapoints, e.g. sample some randomly
-            human_evaluators=self.human_evaluators,
             executor_span_id=executor_span_id,
             index=index,
             metadata=datapoint.metadata,
@@ -394,8 +406,7 @@ class Evaluation:
 def evaluate(
     data: EvaluationDataset | list[Datapoint | dict],
     executor: ExecutorFunction,
-    evaluators: dict[str, EvaluatorFunction],
-    human_evaluators: list[HumanEvaluator] = [],
+    evaluators: dict[str, EvaluatorFunction | HumanEvaluator],
     name: str | None = None,
     group_name: str | None = None,
     concurrency_limit: int = DEFAULT_BATCH_SIZE,
@@ -424,18 +435,15 @@ def evaluate(
         executor (Callable[..., Any]): The executor function.\
             Takes the data point + any additional arguments\
             and returns the output to evaluate.
-        evaluators (List[Callable[..., Any]]): 
-            evaluators (dict[str, Callable[..., Any]]): Evaluator functions and\
-                names. Each evaluator function takes the output of the executor\
-                _and_ the target data, and returns a score. The score can be a\
-                single number or a dict of string keys and number values.\
-                If the score is a single number, it will be named after the\
-                evaluator function. Evaluator function names must contain only\
-                letters, digits, hyphens, underscores, or spaces.
-        human_evaluators (list[HumanEvaluator], optional):\
-            [Beta] List of instances of HumanEvaluator. For now, human\
-            evaluator only holds the queue name.
-            Defaults to an empty list.
+        evaluators (dict[str, Callable[..., Any] | HumanEvaluator]): Evaluator\
+            functions and HumanEvaluator instances with names. Each evaluator\
+            function takes the output of the executor _and_ the target data,\
+            and returns a score. The score can be a single number or a dict\
+            of string keys and number values. If the score is a single number,\
+            it will be named after the evaluator function.\
+            HumanEvaluator instances create empty spans for manual evaluation.\
+            Evaluator function names must contain only letters, digits, hyphens,\
+            underscores, or spaces.
         name (str | None, optional): Optional name of the evaluation.\
             Used to identify the evaluation in the group. If not provided, a\
             random name will be generated.
@@ -470,7 +478,6 @@ def evaluate(
         executor=executor,
         evaluators=evaluators,
         group_name=group_name,
-        human_evaluators=human_evaluators,
         name=name,
         concurrency_limit=concurrency_limit,
         project_api_key=project_api_key,

lmnr/sdk/types.py

@@ -9,7 +9,7 @@ import uuid
 
 from enum import Enum
 from opentelemetry.trace import SpanContext, TraceFlags
-from typing import Any, Awaitable, Callable, Literal
+from typing import Any, Awaitable, Callable, Literal, Optional
 
 from .utils import serialize
 
@@ -50,7 +50,7 @@ EvaluatorFunction = Callable[
 
 
 class HumanEvaluator(pydantic.BaseModel):
-    queueName: str
+    pass
 
 
 class InitEvaluationResponse(pydantic.BaseModel):
@@ -94,8 +94,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     data: EvaluationDatapointData
     target: EvaluationDatapointTarget
     executor_output: ExecutorFunctionReturnType
-    scores: dict[str, Numeric]
-    human_evaluators: list[HumanEvaluator] = pydantic.Field(default_factory=list)
+    scores: dict[str, Optional[Numeric]]
     trace_id: uuid.UUID
     executor_span_id: uuid.UUID
     metadata: EvaluationDatapointMetadata = pydantic.Field(default=None)
@@ -112,14 +111,6 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
                 "executorOutput": str(serialize(self.executor_output))[:100],
                 "scores": self.scores,
                 "traceId": str(self.trace_id),
-                "humanEvaluators": [
-                    (
-                        v.model_dump()
-                        if isinstance(v, pydantic.BaseModel)
-                        else serialize(v)
-                    )
-                    for v in self.human_evaluators
-                ],
                 "executorSpanId": str(self.executor_span_id),
                 "index": self.index,
                 "metadata": (
@@ -136,6 +127,7 @@ class SpanType(Enum):
     PIPELINE = "PIPELINE"  # must not be set manually
     EXECUTOR = "EXECUTOR"
     EVALUATOR = "EVALUATOR"
+    HUMAN_EVALUATOR = "HUMAN_EVALUATOR"
     EVALUATION = "EVALUATION"
 
 

lmnr/version.py

@@ -3,7 +3,7 @@ import httpx
 from packaging import version
 
 
-__version__ = "0.6.11"
+__version__ = "0.6.13"
 PYTHON_VERSION = f"{sys.version_info.major}.{sys.version_info.minor}"
 
 

{lmnr-0.6.11.dist-info → lmnr-0.6.13.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lmnr
-Version: 0.6.11
+Version: 0.6.13
 Summary: Python SDK for Laminar
 License: Apache-2.0
 Author: lmnr.ai
@@ -46,61 +46,61 @@ Requires-Dist: httpx (>=0.25.0)
 Requires-Dist: opentelemetry-api (>=1.33.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.33.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.33.0)
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.8) ; extra == "alephalpha"
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.8) ; extra == "anthropic"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.8) ; extra == "bedrock"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.8) ; extra == "chromadb"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.8) ; extra == "cohere"
-Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.8) ; extra == "crewai"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.8) ; extra == "google-generativeai"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.8) ; extra == "groq"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.8) ; extra == "haystack"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.8) ; extra == "lancedb"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.8) ; extra == "langchain"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.8) ; extra == "llamaindex"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.8) ; extra == "marqo"
-Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.8) ; extra == "mcp"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.8) ; extra == "milvus"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.8) ; extra == "mistralai"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.8) ; extra == "ollama"
-Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.8) ; extra == "openai"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.8) ; extra == "pinecone"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.8) ; extra == "qdrant"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.8) ; extra == "replicate"
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.8) ; extra == "sagemaker"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.12) ; extra == "alephalpha"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.12) ; extra == "anthropic"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.12) ; extra == "bedrock"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.12) ; extra == "chromadb"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.12) ; extra == "cohere"
+Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.12) ; extra == "crewai"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.12) ; extra == "google-generativeai"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.12) ; extra == "groq"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.12) ; extra == "haystack"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.12) ; extra == "lancedb"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.12) ; extra == "langchain"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.12) ; extra == "llamaindex"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.12) ; extra == "marqo"
+Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.12) ; extra == "mcp"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.12) ; extra == "milvus"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.12) ; extra == "mistralai"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.12) ; extra == "ollama"
+Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.12) ; extra == "openai"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.12) ; extra == "pinecone"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.12) ; extra == "qdrant"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.12) ; extra == "replicate"
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.12) ; extra == "sagemaker"
 Requires-Dist: opentelemetry-instrumentation-threading (>=0.54b0)
-Requires-Dist: opentelemetry-instrumentation-together (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-together (>=0.40.8) ; extra == "together"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.8) ; extra == "transformers"
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.8) ; extra == "vertexai"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.8) ; extra == "watsonx"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.8) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.8) ; extra == "weaviate"
+Requires-Dist: opentelemetry-instrumentation-together (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-together (>=0.40.12) ; extra == "together"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.12) ; extra == "transformers"
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.12) ; extra == "vertexai"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.12) ; extra == "watsonx"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.12) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.12) ; extra == "weaviate"
 Requires-Dist: opentelemetry-sdk (>=1.33.0)
 Requires-Dist: opentelemetry-semantic-conventions (>=0.54b0)
 Requires-Dist: opentelemetry-semantic-conventions-ai (>=0.4.8)

{lmnr-0.6.11.dist-info → lmnr-0.6.13.dist-info}/RECORD

@@ -1,11 +1,13 @@
-lmnr/__init__.py,sha256=eJ-gIHEk8KV-BeaU8c9spQww_T2G5_OMu4F8JEzngvA,1281
+lmnr/__init__.py,sha256=gRg8Pb-0XVzC-XmiLlHI9Br5yAJXwT6srPFxXIOaZ1o,1373
 lmnr/cli.py,sha256=uHgLUfN_6eINtUlcQdOtODf2tI9AiwmlhojQF4UMB5Y,6047
 lmnr/opentelemetry_lib/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
-lmnr/opentelemetry_lib/__init__.py,sha256=MHT91gFyPte8OHiy18rQuImy4Bee1DXFSR4CH6-B-Wc,2154
+lmnr/opentelemetry_lib/__init__.py,sha256=aWKsqRXUhVhu2BS555nO2JhZSsK8bTUylAVwWybquGE,2160
 lmnr/opentelemetry_lib/decorators/__init__.py,sha256=45HVoYnHC1Y9D_VSkioDbqD3gm4RPC5sKoztomBI5j8,8496
-lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py,sha256=6Fvkc_zZEX1lk8g6ZGFrADLNOL055pkMdO-hEef8qBY,18525
+lmnr/opentelemetry_lib/litellm/__init__.py,sha256=wjo46It5GdhmxPCaiA8kaKyaz3VsuRDRYUCCstvK0-Y,14858
+lmnr/opentelemetry_lib/litellm/utils.py,sha256=2ozwVT-C3HIDEJ8Rekx7QYXouvNMqtEteCOHVRUgGic,539
+lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py,sha256=mltiTDVCCyMuhQNuoLHvblg9O5X0ncG6xN3f1opSeQU,18613
 lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/config.py,sha256=25zevJ7g3MtJP_5gju3jBH7-wg7SbDkktysuUO29ksI,245
-lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/utils.py,sha256=ICQENOiICTKodjZVHhq3H5RIRY5bbuWp_KmzkDNgDRM,7471
+lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/utils.py,sha256=8SSBliRoJtiZME5RDEwt90CI2BadKPHQrtV4p6bDy_0,7669
 lmnr/opentelemetry_lib/opentelemetry/instrumentation/langgraph/__init__.py,sha256=Jyv9koZdGA4-oTaB7ATB7DaX7aNOY-3YOGL4wX0c7PM,3107
 lmnr/opentelemetry_lib/opentelemetry/instrumentation/langgraph/utils.py,sha256=nf9sJZXnnts4gYZortEiDvwYjYqYJZTAT0zutuP_R6Y,1512
 lmnr/opentelemetry_lib/tracing/__init__.py,sha256=27QogAe-aHyrVr6b56-DduUm0KEE24K1aV2e1nouTNg,6007
@@ -45,14 +47,14 @@ lmnr/sdk/client/synchronous/sync_client.py,sha256=IIzj-mAwHHoRuUX9KkJtrzTGi5UOyg
 lmnr/sdk/datasets.py,sha256=P9hRxfl7-I6qhLFFGgU-r_I7RJfLtF6sL56g5fKIbAA,1708
 lmnr/sdk/decorators.py,sha256=1uu9xxBYgblFqlhQqH17cZYq7babAmB1lEtvBgTsP0E,4468
 lmnr/sdk/eval_control.py,sha256=KROUrDhcZTrptRZ-hxvr60_o_Gt_8u045jb4cBXcuoY,184
-lmnr/sdk/evaluations.py,sha256=i5c9wi9ZWV-L7vYbHEnLQC2V34n3tasPRowJAnSr-qQ,21022
+lmnr/sdk/evaluations.py,sha256=fMUDueAgGv9fyTuX7n0DsS8lOzrbZNMgPorA037tgDU,21458
 lmnr/sdk/laminar.py,sha256=oOVco_c9ZstT71HsquGsgbtFumXd2Ejz0rl_qpmMlTU,33996
 lmnr/sdk/log.py,sha256=nt_YMmPw1IRbGy0b7q4rTtP4Yo3pQfNxqJPXK3nDSNQ,2213
-lmnr/sdk/types.py,sha256=5tEX7yoemb9wYyXLy4aqdazudO5I8dglU5A-IegDhsQ,12653
+lmnr/sdk/types.py,sha256=ZQp5SeYJNZsK3KrbSeXPY_xn6mGjW5mSw_i0Rd_Oa4k,12328
 lmnr/sdk/utils.py,sha256=yrcHIhoADf9lWH9qJWZMmkRWYvd0DuxPSLP3mY6YFw0,4327
-lmnr/version.py,sha256=qnwKwsPBqcM4aZx6FBz8GRvhfVyvAhaXzQqQSfXM-k0,1322
-lmnr-0.6.11.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.6.11.dist-info/METADATA,sha256=DpYsYjFiUQII2I71j7YJv6F10rgBqGsHCfG_lLQQfhQ,15132
-lmnr-0.6.11.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-lmnr-0.6.11.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.6.11.dist-info/RECORD,,
+lmnr/version.py,sha256=9gMGnzWiCqbv_PzYX6c-jQFz17zO1xyIONO32wE9xfY,1322
+lmnr-0.6.13.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.6.13.dist-info/METADATA,sha256=95yHwJlTi4CNL1ucZDk9ge2flO712qjpxikJ0jCkmiI,15186
+lmnr-0.6.13.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+lmnr-0.6.13.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.6.13.dist-info/RECORD,,