lmnr 0.6.19__py3-none-any.whl → 0.6.21__py3-none-any.whl

lmnr/sdk/client/asynchronous/resources/evaluators.py

@@ -0,0 +1,85 @@
+"""Evaluators resource for creating evaluator scores."""
+
+import uuid
+from typing import Any
+
+from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
+from lmnr.sdk.utils import format_id
+
+
+class AsyncEvaluators(BaseAsyncResource):
+    """Resource for creating evaluator scores."""
+
+    async def score(
+        self,
+        *,
+        name: str,
+        trace_id: str | int | uuid.UUID | None = None,
+        span_id: str | int | uuid.UUID | None = None,
+        metadata: dict[str, Any] | None = None,
+        score: float,
+    ) -> None:
+        """Create a score for a span.
+
+        Args:
+            name (str): Name of the score
+            trace_id (str | int | uuid.UUID | None, optional): The trace ID to score (will be attached to root span)
+            span_id (str | int | uuid.UUID | None, optional): The span ID to score
+            metadata (dict[str, Any] | None, optional): Additional metadata. Defaults to None.
+            score (float): The score value (float)
+
+        Raises:
+            ValueError: If there's an error creating the score.
+
+        Example:
+            Score by trace ID (will attach to root span):
+
+            >>> await laminar_client.evaluators.score(
+            ...     name="quality",
+            ...     trace_id="trace-id-here",
+            ...     score=0.95,
+            ...     metadata={"model": "gpt-4"}
+            ... )
+
+            Score by span ID:
+
+            >>> await laminar_client.evaluators.score(
+            ...     name="relevance",
+            ...     span_id="span-id-here",
+            ...     score=0.87
+            ... )
+        """
+        if trace_id is not None and span_id is not None:
+            raise ValueError("Cannot provide both trace_id and span_id. Please provide only one.")
+        if trace_id is None and span_id is None:
+            raise ValueError("Either 'trace_id' or 'span_id' must be provided.")
+
+        if trace_id is not None:
+            formatted_trace_id = format_id(trace_id)
+            payload = {
+                "name": name,
+                "traceId": formatted_trace_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+        else:
+            formatted_span_id = format_id(span_id)
+            payload = {
+                "name": name,
+                "spanId": formatted_span_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+
+        response = await self._client.post(
+            self._base_url + "/v1/evaluators/score",
+            json=payload,
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error creating evaluator score: {response.text}")

lmnr/sdk/client/asynchronous/resources/tags.py

@@ -5,6 +5,7 @@ import uuid
 
 from lmnr.sdk.client.asynchronous.resources.base import BaseAsyncResource
 from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.utils import format_id
 
 logger = get_default_logger(__name__)
 
@@ -54,18 +55,11 @@ class AsyncTags(BaseAsyncResource):
         ```
         """
         trace_tags = tags if isinstance(tags, list) else [tags]
-        if isinstance(trace_id, uuid.UUID):
-            trace_id = str(trace_id)
-        elif isinstance(trace_id, int):
-            trace_id = str(uuid.UUID(int=trace_id))
-        elif isinstance(trace_id, str):
-            uuid.UUID(trace_id)  # Will raise ValueError if invalid
-        else:
-            raise ValueError(f"Invalid trace id: {trace_id}")
+        formatted_trace_id = format_id(trace_id)
 
         url = self._base_url + "/v1/tag"
         payload = {
-            "traceId": trace_id,
+            "traceId": formatted_trace_id,
             "names": trace_tags,
         }
         response = await self._client.post(
@@ -78,7 +72,7 @@ class AsyncTags(BaseAsyncResource):
 
         if response.status_code == 404:
             logger.warning(
-                f"Trace {trace_id} not found. The trace may have not been ended yet."
+                f"Trace {formatted_trace_id} not found. The trace may have not been ended yet."
             )
             return []
 

lmnr/sdk/client/synchronous/resources/__init__.py

@@ -2,5 +2,6 @@ from lmnr.sdk.client.synchronous.resources.agent import Agent
 from lmnr.sdk.client.synchronous.resources.browser_events import BrowserEvents
 from lmnr.sdk.client.synchronous.resources.evals import Evals
 from lmnr.sdk.client.synchronous.resources.tags import Tags
+from lmnr.sdk.client.synchronous.resources.evaluators import Evaluators
 
-__all__ = ["Agent", "Evals", "BrowserEvents", "Tags"]
+__all__ = ["Agent", "Evals", "Evaluators", "BrowserEvents", "Tags"]

lmnr/sdk/client/synchronous/resources/evaluators.py

@@ -0,0 +1,85 @@
+"""Evaluators resource for creating evaluator scores."""
+
+import uuid
+from typing import Any
+
+from lmnr.sdk.client.synchronous.resources.base import BaseResource
+from lmnr.sdk.utils import format_id
+
+
+class Evaluators(BaseResource):
+    """Resource for creating evaluator scores."""
+
+    def score(
+        self,
+        *,
+        name: str,
+        trace_id: str | int | uuid.UUID | None = None,
+        span_id: str | int | uuid.UUID | None = None,
+        metadata: dict[str, Any] | None = None,
+        score: float,
+    ) -> None:
+        """Create a score for a span.
+
+        Args:
+            name (str): Name of the score
+            trace_id (str | int | uuid.UUID | None, optional): The trace ID to score (will be attached to root span)
+            span_id (str | int | uuid.UUID | None, optional): The span ID to score
+            metadata (dict[str, Any] | None, optional): Additional metadata. Defaults to None.
+            score (float): The score value (float)
+
+        Raises:
+            ValueError: If there's an error creating the score.
+
+        Example:
+            Score by trace ID (will attach to root span):
+
+            >>> laminar_client.evaluators.score(
+            ...     name="quality",
+            ...     trace_id="trace-id-here",
+            ...     score=0.95,
+            ...     metadata={"model": "gpt-4"}
+            ... )
+
+            Score by span ID:
+
+            >>> laminar_client.evaluators.score(
+            ...     name="relevance",
+            ...     span_id="span-id-here", 
+            ...     score=0.87
+            ... )
+        """
+        if trace_id is not None and span_id is not None:
+            raise ValueError("Cannot provide both trace_id and span_id. Please provide only one.")
+        if trace_id is None and span_id is None:
+            raise ValueError("Either 'trace_id' or 'span_id' must be provided.")
+
+        if trace_id is not None:
+            formatted_trace_id = format_id(trace_id)
+            payload = {
+                "name": name,
+                "traceId": formatted_trace_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+        else:
+            formatted_span_id = format_id(span_id)
+            payload = {
+                "name": name,
+                "spanId": formatted_span_id,
+                "metadata": metadata,
+                "score": score,
+                "source": "Code",
+            }
+
+        response = self._client.post(
+            self._base_url + "/v1/evaluators/score",
+            json=payload,
+            headers=self._headers(),
+        )
+
+        if response.status_code != 200:
+            if response.status_code == 401:
+                raise ValueError("Unauthorized. Please check your project API key.")
+            raise ValueError(f"Error creating evaluator score: {response.text}")

lmnr/sdk/client/synchronous/resources/tags.py

@@ -5,6 +5,7 @@ import uuid
 
 from lmnr.sdk.client.synchronous.resources.base import BaseResource
 from lmnr.sdk.log import get_default_logger
+from lmnr.sdk.utils import format_id
 
 logger = get_default_logger(__name__)
 
@@ -54,18 +55,11 @@ class Tags(BaseResource):
         ```
         """
         trace_tags = tags if isinstance(tags, list) else [tags]
-        if isinstance(trace_id, uuid.UUID):
-            trace_id = str(trace_id)
-        elif isinstance(trace_id, int):
-            trace_id = str(uuid.UUID(int=trace_id))
-        elif isinstance(trace_id, str):
-            uuid.UUID(trace_id)
-        else:
-            raise ValueError(f"Invalid trace id: {trace_id}")
+        formatted_trace_id = format_id(trace_id)
 
         url = self._base_url + "/v1/tag"
         payload = {
-            "traceId": trace_id,
+            "traceId": formatted_trace_id,
             "names": trace_tags,
         }
         response = self._client.post(
@@ -78,7 +72,7 @@ class Tags(BaseResource):
 
         if response.status_code == 404:
             logger.warning(
-                f"Trace {trace_id} not found. The trace may have not been ended yet."
+                f"Trace {formatted_trace_id} not found. The trace may have not been ended yet."
             )
             return []
 

lmnr/sdk/client/synchronous/sync_client.py

@@ -11,6 +11,7 @@ from lmnr.sdk.client.synchronous.resources import (
     Agent,
     BrowserEvents,
     Evals,
+    Evaluators,
     Tags,
 )
 from lmnr.sdk.utils import from_env
@@ -27,6 +28,7 @@ class LaminarClient:
     __agent: Agent | None = None
     __evals: Evals | None = None
     __tags: Tags | None = None
+    __evaluators: Evaluators | None = None
 
     def __init__(
         self,
@@ -74,6 +76,9 @@ class LaminarClient:
         # Initialize resource objects
         self.__agent = Agent(self.__client, self.__base_url, self.__project_api_key)
         self.__evals = Evals(self.__client, self.__base_url, self.__project_api_key)
+        self.__evaluators = Evaluators(
+            self.__client, self.__base_url, self.__project_api_key
+        )
         self.__browser_events = BrowserEvents(
             self.__client, self.__base_url, self.__project_api_key
         )
@@ -115,6 +120,15 @@ class LaminarClient:
         """
         return self.__tags
 
+    @property
+    def evaluators(self) -> Evaluators:
+        """Get the Evaluators resource.
+
+        Returns:
+            Evaluators: The Evaluators resource instance.
+        """
+        return self.__evaluators
+
     def shutdown(self):
         """Shutdown the client by closing underlying connections."""
         self.__client.close()

lmnr/sdk/decorators.py

@@ -1,6 +1,6 @@
 from lmnr.opentelemetry_lib.decorators import (
-    entity_method,
-    aentity_method,
+    observe_base,
+    async_observe_base,
     json_dumps,
 )
 from opentelemetry.trace import INVALID_SPAN, get_current_span
@@ -28,6 +28,8 @@ def observe(
     ignore_output: bool = False,
     span_type: Literal["DEFAULT", "LLM", "TOOL"] = "DEFAULT",
     ignore_inputs: list[str] | None = None,
+    input_formatter: Callable[P, str] | None = None,
+    output_formatter: Callable[[R], str] | None = None,
     metadata: dict[str, Any] | None = None,
     tags: list[str] | None = None,
 ) -> Callable[[Callable[P, R]], Callable[P, R]]:
@@ -53,6 +55,16 @@ def observe(
             def foo(a, b, `sensitive_data`), and you want to ignore the\
             `sensitive_data` argument, you can pass ["sensitive_data"] to\
             this argument. Defaults to None.
+        input_formatter (Callable[P, str] | None, optional): A custom function\
+            to format the input of the wrapped function. All function arguments\
+            are passed to this function. Must return a string. Ignored if\
+            `ignore_input` is True. Does not respect `ignore_inputs` argument.
+            Defaults to None.
+        output_formatter (Callable[[R], str] | None, optional): A custom function\
+            to format the output of the wrapped function. The output is passed\
+            to this function. Must return a string. Ignored if `ignore_output`
+            is True. Does not respect `ignore_inputs` argument.
+            Defaults to None.
         metadata (dict[str, Any] | None, optional): Metadata to associate with\
             the trace. Must be JSON serializable. Defaults to None.
         tags (list[str] | None, optional): Tags to associate with the trace.
@@ -91,22 +103,45 @@ def observe(
                 logger.warning("Tags must be a list of strings. Tags will be ignored.")
             else:
                 association_properties["tags"] = tags
+        if input_formatter is not None and ignore_input:
+            logger.warning(
+                f"observe, function {func.__name__}: Input formatter"
+                " is ignored because `ignore_input` is True. Specify only one of"
+                " `ignore_input` or `input_formatter`."
+            )
+        if input_formatter is not None and ignore_inputs is not None:
+            logger.warning(
+                f"observe, function {func.__name__}: Both input formatter and"
+                " `ignore_inputs` are specified. Input formatter"
+                " will pass all arguments to the formatter regardless of"
+                " `ignore_inputs`."
+            )
+        if output_formatter is not None and ignore_output:
+            logger.warning(
+                f"observe, function {func.__name__}: Output formatter"
+                " is ignored because `ignore_output` is True. Specify only one of"
+                " `ignore_output` or `output_formatter`."
+            )
         result = (
-            aentity_method(
+            async_observe_base(
                 name=name,
                 ignore_input=ignore_input,
                 ignore_output=ignore_output,
                 span_type=span_type,
                 ignore_inputs=ignore_inputs,
+                input_formatter=input_formatter,
+                output_formatter=output_formatter,
                 association_properties=association_properties,
             )(func)
             if is_async(func)
-            else entity_method(
+            else observe_base(
                 name=name,
                 ignore_input=ignore_input,
                 ignore_output=ignore_output,
                 span_type=span_type,
                 ignore_inputs=ignore_inputs,
+                input_formatter=input_formatter,
+                output_formatter=output_formatter,
                 association_properties=association_properties,
             )(func)
         )

lmnr/sdk/evaluations.py

@@ -57,7 +57,7 @@ def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Nu
     average_scores = {}
     for key, values in per_score_values.items():
         scores = [v for v in values if v is not None]
-        
+
         # If there are no scores, we don't want to include the key in the average scores
         if len(scores) > 0:
             average_scores[key] = sum(scores) / len(scores)
@@ -108,6 +108,7 @@ class Evaluation:
         concurrency_limit: int = DEFAULT_BATCH_SIZE,
         project_api_key: str | None = None,
         base_url: str | None = None,
+        base_http_url: str | None = None,
         http_port: int | None = None,
         grpc_port: int | None = None,
         instruments: set[Instruments] | None = None,
@@ -157,6 +158,10 @@ class Evaluation:
                 Useful if self-hosted. Do NOT include the port, use `http_port`\
                 and `grpc_port` instead.
                 Defaults to "https://api.lmnr.ai".
+            base_http_url (str | None, optional): The base HTTP URL for Laminar API.\
+                Only set this if your Laminar backend HTTP is proxied\
+                through a different host. If not specified, defaults\
+                to https://api.lmnr.ai.
             http_port (int | None, optional): The port for Laminar API\
                 HTTP service. Defaults to 443 if not specified.
             grpc_port (int | None, optional): The port for Laminar API\
@@ -199,7 +204,7 @@ class Evaluation:
         self.batch_size = concurrency_limit
         self._logger = get_default_logger(self.__class__.__name__)
         self.upload_tasks = []
-        self.base_http_url = f"{base_url}:{http_port or 443}"
+        self.base_http_url = f"{base_http_url or base_url}:{http_port or 443}"
 
         api_key = project_api_key or from_env("LMNR_PROJECT_API_KEY")
         if not api_key and not L.is_initialized():
@@ -224,6 +229,7 @@ class Evaluation:
         L.initialize(
             project_api_key=project_api_key,
             base_url=base_url,
+            base_http_url=self.base_http_url,
             http_port=http_port,
             grpc_port=grpc_port,
             instruments=instruments,
@@ -352,22 +358,24 @@ class Evaluation:
                 if isinstance(evaluator, HumanEvaluator):
                     # Create an empty span for human evaluators
                     with L.start_as_current_span(
-                        evaluator_name,
-                        input={"output": output, "target": target}
+                        evaluator_name, input={"output": output, "target": target}
                     ) as human_evaluator_span:
-                        human_evaluator_span.set_attribute(SPAN_TYPE, SpanType.HUMAN_EVALUATOR.value)
+                        human_evaluator_span.set_attribute(
+                            SPAN_TYPE, SpanType.HUMAN_EVALUATOR.value
+                        )
                         # Human evaluators don't execute automatically, just create the span
                         L.set_span_output(None)
-                    
+
                     # We don't want to save the score for human evaluators
                     scores[evaluator_name] = None
                 else:
                     # Regular evaluator function
                     with L.start_as_current_span(
-                        evaluator_name,
-                        input={"output": output, "target": target}
+                        evaluator_name, input={"output": output, "target": target}
                     ) as evaluator_span:
-                        evaluator_span.set_attribute(SPAN_TYPE, SpanType.EVALUATOR.value)
+                        evaluator_span.set_attribute(
+                            SPAN_TYPE, SpanType.EVALUATOR.value
+                        )
                         if is_async(evaluator):
                             value = await evaluator(output, target)
                         else:
@@ -416,6 +424,7 @@ def evaluate(
     concurrency_limit: int = DEFAULT_BATCH_SIZE,
     project_api_key: str | None = None,
     base_url: str | None = None,
+    base_http_url: str | None = None,
     http_port: int | None = None,
     grpc_port: int | None = None,
     instruments: set[Instruments] | None = None,
@@ -465,6 +474,10 @@ def evaluate(
                         Useful if self-hosted elsewhere. Do NOT include the\
                         port, use `http_port` and `grpc_port` instead.
                         Defaults to "https://api.lmnr.ai".
+        base_http_url (str | None, optional): The base HTTP URL for Laminar API.\
+                        Only set this if your Laminar backend HTTP is proxied\
+                        through a different host. If not specified, defaults\
+                        to https://api.lmnr.ai.
         http_port (int | None, optional): The port for Laminar API's HTTP\
                         service. 443 is used if not specified.
                         Defaults to None.
@@ -488,6 +501,7 @@ def evaluate(
         concurrency_limit=concurrency_limit,
         project_api_key=project_api_key,
         base_url=base_url,
+        base_http_url=base_http_url,
         http_port=http_port,
         grpc_port=grpc_port,
         instruments=instruments,