lmnr 0.4.23__py3-none-any.whl → 0.4.24__py3-none-any.whl

lmnr/__init__.py

@@ -1,3 +1,4 @@
+from .sdk.datasets import EvaluationDataset, LaminarDataset
 from .sdk.evaluations import evaluate
 from .sdk.laminar import Laminar
 from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput

lmnr/sdk/datasets.py

@@ -0,0 +1,58 @@
+from abc import ABC, abstractmethod
+import logging
+
+from .log import get_default_logger
+from .laminar import Laminar as L
+from .types import (
+    Datapoint,
+)
+
+DEFAULT_FETCH_SIZE = 25
+
+
+class EvaluationDataset(ABC):
+    @abstractmethod
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def __len__(self) -> int:
+        pass
+
+    @abstractmethod
+    def __getitem__(self, idx) -> Datapoint:
+        pass
+
+    def slice(self, start: int, end: int):
+        return [self[i] for i in range(max(start, 0), min(end, len(self)))]
+
+
+class LaminarDataset(EvaluationDataset):
+    def __init__(self, name: str, fetch_size: int = DEFAULT_FETCH_SIZE):
+        self.name = name
+        self._len = None
+        self._fetched_items = []
+        self._offset = 0
+        self._fetch_size = fetch_size
+        self._logger = get_default_logger(self.__class__.__name__, level=logging.DEBUG)
+
+    def _fetch_batch(self):
+        self._logger.debug(
+            f"dataset {self.name}. Fetching batch from {self._offset} to "
+            + f"{self._offset + self._fetch_size}"
+        )
+        resp = L.get_datapoints(self.name, self._offset, self._fetch_size)
+        self._fetched_items += resp.items
+        self._offset = len(self._fetched_items)
+        if self._len is None:
+            self._len = resp.totalCount
+
+    def __len__(self) -> int:
+        if self._len is None:
+            self._fetch_batch()
+        return self._len
+
+    def __getitem__(self, idx) -> Datapoint:
+        if idx >= len(self._fetched_items):
+            self._fetch_batch()
+        return self._fetched_items[idx]

lmnr/sdk/evaluations.py

@@ -1,17 +1,18 @@
 import asyncio
 import re
 import sys
-from abc import ABC, abstractmethod
-from contextlib import contextmanager
-from typing import Any, Awaitable, Optional, Set, Union
 import uuid
 
+from contextlib import contextmanager
 from tqdm import tqdm
+from typing import Any, Awaitable, Optional, Set, Union
 
 from ..traceloop_sdk.instruments import Instruments
 from ..traceloop_sdk.tracing.attributes import SPAN_TYPE
 
+from .datasets import EvaluationDataset
 from .laminar import Laminar as L
+from .log import get_default_logger
 from .types import (
     Datapoint,
     EvaluationResultDatapoint,
@@ -84,7 +85,7 @@ class EvaluationReporter:
     ):
         self.cli_progress.close()
         print(
-            f"\nCheck progress and results at {get_evaluation_url(project_id, evaluation_id)}\n"
+            f"\nCheck the results at {get_evaluation_url(project_id, evaluation_id)}\n"
         )
         print("Average scores:")
         for name, score in average_scores.items():
@@ -92,23 +93,6 @@ class EvaluationReporter:
         print("\n")
 
 
-class EvaluationDataset(ABC):
-    @abstractmethod
-    def __init__(self, *args, **kwargs):
-        pass
-
-    @abstractmethod
-    def __len__(self) -> int:
-        pass
-
-    @abstractmethod
-    def __getitem__(self, idx) -> Datapoint:
-        pass
-
-    def slice(self, start: int, end: int):
-        return [self[i] for i in range(max(start, 0), min(end, len(self)))]
-
-
 class Evaluation:
     def __init__(
         self,
@@ -135,14 +119,13 @@ class Evaluation:
             executor (Callable[..., Any]): The executor function.\
                             Takes the data point + any additional arguments\
                             and returns the output to evaluate.
-            evaluators (List[Callable[..., Any]]): List of evaluator functions.\
-                Each evaluator function takes the output of the executor _and_\
-                the target data, and returns a score. The score can be a\
-                single number or a record of string keys and number values.\
+            evaluators (dict[str, Callable[..., Any]]): Evaluator functions and\
+                names. Each evaluator function takes the output of the executor\
+                _and_ the target data, and returns a score. The score can be a\
+                single number or a dict of string keys and number values.\
                 If the score is a single number, it will be named after the\
-                evaluator function. If the function is anonymous, it will be\
-                named `evaluator_${index}`, where index is the index of the\
-                evaluator function in the list starting from 1.
+                evaluator function. Evaluator function names must contain only\
+                letters, digits, hyphens, underscores, or spaces.
             group_id (Optional[str], optional): Group id of the evaluation.
                             Defaults to "default".
             name (Optional[str], optional): The name of the evaluation.\
@@ -191,6 +174,7 @@ class Evaluation:
         self.group_id = group_id
         self.name = name
         self.batch_size = batch_size
+        self._logger = get_default_logger(self.__class__.__name__)
         L.initialize(
             project_api_key=project_api_key,
             base_url=base_url,
@@ -215,7 +199,7 @@ class Evaluation:
         )
 
         try:
-            result_datapoints = await self.evaluate_in_batches()
+            result_datapoints = await self._evaluate_in_batches()
         except Exception as e:
             self.reporter.stopWithError(e)
             self.is_finished = True
@@ -228,7 +212,7 @@ class Evaluation:
             self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
             self.is_finished = True
 
-    async def evaluate_in_batches(self) -> list[EvaluationResultDatapoint]:
+    async def _evaluate_in_batches(self) -> list[EvaluationResultDatapoint]:
         result_datapoints = []
         for i in range(0, len(self.data), self.batch_size):
             batch = (
@@ -326,14 +310,14 @@ def evaluate(
         executor (Callable[..., Any]): The executor function.\
                         Takes the data point + any additional arguments\
                         and returns the output to evaluate.
-        evaluators (List[Callable[..., Any]]): List of evaluator functions.\
-            Each evaluator function takes the output of the executor _and_\
-            the target data, and returns a score. The score can be a\
-            single number or a record of string keys and number values.\
-            If the score is a single number, it will be named after the\
-            evaluator function. If the function is anonymous, it will be\
-            named `evaluator_${index}`, where index is the index of the\
-            evaluator function in the list starting from 1.
+        evaluators (List[Callable[..., Any]]): 
+            evaluators (dict[str, Callable[..., Any]]): Evaluator functions and\
+                names. Each evaluator function takes the output of the executor\
+                _and_ the target data, and returns a score. The score can be a\
+                single number or a dict of string keys and number values.\
+                If the score is a single number, it will be named after the\
+                evaluator function. Evaluator function names must contain only\
+                letters, digits, hyphens, underscores, or spaces.
         group_id (Optional[str], optional): an identifier to group evaluations.\
                         It is practical to group evaluations that evaluate\
                         the same feature on the same dataset, to be able to\

lmnr/sdk/laminar.py

@@ -24,6 +24,7 @@ import json
 import logging
 import os
 import requests
+import urllib.parse
 import uuid
 
 from lmnr.traceloop_sdk.tracing.attributes import (
@@ -45,6 +46,7 @@ from .log import VerboseColorfulFormatter
 from .types import (
     CreateEvaluationResponse,
     EvaluationResultDatapoint,
+    GetDatapointsResponse,
     PipelineRunError,
     PipelineRunResponse,
     NodeInput,
@@ -284,7 +286,9 @@ class Laminar:
         span_type: Union[Literal["DEFAULT"], Literal["LLM"]] = "DEFAULT",
     ):
         """Start a new span as the current span. Useful for manual
-        instrumentation.
+        instrumentation. If `span_type` is set to `"LLM"`, you should report
+        usage and response attributes manually. See `Laminar.set_span_attributes`
+        for more information.
 
         Usage example:
         ```python
@@ -297,6 +301,9 @@ class Laminar:
             name (str): name of the span
             input (Any, optional): input to the span. Will be sent as an\
                 attribute, so must be json serializable. Defaults to None.
+            span_type (Union[Literal["DEFAULT"], Literal["LLM"]], optional):\
+                type of the span. If you use `"LLM"`, you should report usage\
+                and response attributes manually. Defaults to "DEFAULT".
         """
         with get_tracer() as tracer:
             span_path = get_span_path(name)
@@ -341,6 +348,22 @@ class Laminar:
     ):
         """Set attributes for the current span. Useful for manual
         instrumentation.
+        Example:
+        ```python
+        with L.start_as_current_span(
+            name="my_span_name", input=input["messages"], span_type="LLM"
+        ):
+            response = await my_custom_call_to_openai(input)
+            L.set_span_output(response["choices"][0]["message"]["content"])
+            L.set_span_attributes({
+                Attributes.PROVIDER: 'openai',
+                Attributes.REQUEST_MODEL: input["model"],
+                Attributes.RESPONSE_MODEL: response["model"],
+                Attributes.INPUT_TOKEN_COUNT: response["usage"]["prompt_tokens"],
+                Attributes.OUTPUT_TOKEN_COUNT: response["usage"]["completion_tokens"],
+            })
+            # ...
+        ```
 
         Args:
             attributes (dict[ATTRIBUTES, Any]): attributes to set for the span
@@ -433,10 +456,36 @@ class Laminar:
             try:
                 resp_json = response.json()
                 raise ValueError(f"Error creating evaluation {json.dumps(resp_json)}")
-            except Exception:
+            except requests.exceptions.RequestException:
                 raise ValueError(f"Error creating evaluation {response.text}")
         return CreateEvaluationResponse.model_validate(response.json())
 
+    @classmethod
+    def get_datapoints(
+        cls,
+        dataset_name: str,
+        offset: int,
+        limit: int,
+    ) -> GetDatapointsResponse:
+        params = {"name": dataset_name, "offset": offset, "limit": limit}
+        url = (
+            cls.__base_http_url
+            + "/v1/datasets/datapoints?"
+            + urllib.parse.urlencode(params)
+        )
+        response = requests.get(url, headers=cls._headers())
+        if response.status_code != 200:
+            try:
+                resp_json = response.json()
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {json.dumps(resp_json)}"
+                )
+            except requests.exceptions.RequestException:
+                raise ValueError(
+                    f"Error fetching datapoints: [{response.status_code}] {response.text}"
+                )
+        return GetDatapointsResponse.model_validate(response.json())
+
     @classmethod
     def _headers(cls):
         assert cls.__project_api_key is not None, "Project API key is not set"

lmnr/sdk/log.py

@@ -37,3 +37,13 @@ class VerboseFormatter(CustomFormatter):
     def format(self, record):
         formatter = logging.Formatter(self.fmt)
         return formatter.format(record)
+
+
+def get_default_logger(name: str, level: int = logging.INFO, propagate: bool = False):
+    logger = logging.getLogger(name)
+    logger.setLevel(level)
+    console_log_handler = logging.StreamHandler()
+    console_log_handler.setFormatter(VerboseColorfulFormatter())
+    logger.addHandler(console_log_handler)
+    logger.propagate = propagate
+    return logger

lmnr/sdk/types.py

@@ -79,6 +79,7 @@ class PipelineRunError(Exception):
 
 EvaluationDatapointData = dict[str, Any]
 EvaluationDatapointTarget = dict[str, Any]
+EvaluationDatapointMetadata = Optional[dict[str, Any]]
 
 
 # EvaluationDatapoint is a single data point in the evaluation
@@ -88,6 +89,7 @@ class Datapoint(pydantic.BaseModel):
     # input to the evaluator function (alongside the executor output).
     # Must be a dict with string keys
     target: EvaluationDatapointTarget
+    metadata: EvaluationDatapointMetadata = pydantic.Field(default=None)
 
 
 ExecutorFunctionReturnType = Any
@@ -153,3 +155,8 @@ class TraceType(Enum):
     DEFAULT = "DEFAULT"
     EVENT = "EVENT"  # must not be set manually
     EVALUATION = "EVALUATION"
+
+
+class GetDatapointsResponse(pydantic.BaseModel):
+    items: list[Datapoint]
+    totalCount: int

lmnr/traceloop_sdk/decorators/base.py

@@ -1,10 +1,10 @@
 import json
 from functools import wraps
+import logging
 import os
 import pydantic
 import types
 from typing import Any, Optional
-import warnings
 
 from opentelemetry import trace
 from opentelemetry import context as context_api
@@ -28,12 +28,10 @@ class CustomJSONEncoder(JSONEncoder):
 
 def json_dumps(data: dict) -> str:
     try:
-        with warnings.catch_warnings():
-            warnings.simplefilter("ignore", RuntimeWarning)
-            return json.dumps(data, cls=CustomJSONEncoder)
+        return json.dumps(data, cls=CustomJSONEncoder)
     except Exception:
         # Log the exception and return a placeholder if serialization completely fails
-        # Telemetry().log_exception(e)
+        logging.warning("Failed to serialize data to JSON, type: %s", type(data))
         return "{}"  # Return an empty JSON object as a fallback
 
 
@@ -141,7 +139,7 @@ def aentity_method(
 
                 try:
                     if _should_send_prompts():
-                        span.set_attribute(SPAN_OUTPUT, json.dumps(res))
+                        span.set_attribute(SPAN_OUTPUT, json_dumps(res))
                 except TypeError:
                     pass
 

lmnr/traceloop_sdk/tracing/attributes.py

@@ -14,10 +14,22 @@ TRACE_TYPE = "trace_type"
 
 # exposed to the user, configurable
 class Attributes(Enum):
+    # == This is the minimum set of attributes for a proper LLM span ==
+    #
     # not SpanAttributes.LLM_USAGE_PROMPT_TOKENS,
     INPUT_TOKEN_COUNT = "gen_ai.usage.input_tokens"
     # not SpanAttributes.LLM_USAGE_COMPLETION_TOKENS,
     OUTPUT_TOKEN_COUNT = "gen_ai.usage.output_tokens"
+    TOTAL_TOKEN_COUNT = SpanAttributes.LLM_USAGE_TOTAL_TOKENS
     PROVIDER = SpanAttributes.LLM_SYSTEM
     REQUEST_MODEL = SpanAttributes.LLM_REQUEST_MODEL
     RESPONSE_MODEL = SpanAttributes.LLM_RESPONSE_MODEL
+    #
+    ## == End of minimum set ==
+    # == Additional attributes ==
+    #
+    INPUT_COST = "gen_ai.usage.input_cost"
+    OUTPUT_COST = "gen_ai.usage.output_cost"
+    TOTAL_COST = "gen_ai.usage.cost"
+    #
+    # == End of additional attributes ==

{lmnr-0.4.23.dist-info → lmnr-0.4.24.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.23
+Version: 0.4.24
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -16,6 +16,7 @@ Requires-Dist: argparse (>=1.0,<2.0)
 Requires-Dist: backoff (>=2.0,<3.0)
 Requires-Dist: deprecated (>=1.0,<2.0)
 Requires-Dist: jinja2 (>=3.0,<4.0)
+Requires-Dist: openai (>=1.52.0,<2.0.0)
 Requires-Dist: opentelemetry-api (>=1.27.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.27.0,<2.0.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.27.0,<2.0.0)

{lmnr-0.4.23.dist-info → lmnr-0.4.24.dist-info}/RECORD

@@ -1,18 +1,19 @@
-lmnr/__init__.py,sha256=-MdXpUkDNhqJaQ5AVdBohIDQgGmeov_xwwtwErwXnQg,288
+lmnr/__init__.py,sha256=87ZHKNA2xmZvqtXrWjhXUNR7YUrdP8-77fYqe-PEtrE,348
 lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lmnr/sdk/datasets.py,sha256=V6q0Zcv7znx5cB1YZd0pBgTEcAKXbb-pEwguScEhBw0,1612
 lmnr/sdk/decorators.py,sha256=ZSDaEZyjo-RUzRCltsNbe6x0t9SKl2xRQ2q4uaKvXtk,2250
-lmnr/sdk/evaluations.py,sha256=C2lhrevH3PDvnTX-qfop1k-NTqTFX4MWaSkc9oG5kxc,15199
-lmnr/sdk/laminar.py,sha256=IoQ_4DJGysKn7tEFZk3ILdRWGxmHvDPzxzXgNZifyws,16293
-lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
-lmnr/sdk/types.py,sha256=HvaZEqVRduCZbkF7Cp8rgS5oBbc1qPvOD3PP9tFrRu4,4826
+lmnr/sdk/evaluations.py,sha256=XzJDRAXmLWxhX3Px0WKocNQYWRZMkI2bG-_Hb054lvA,14874
+lmnr/sdk/laminar.py,sha256=HLnXd7rKm8LlLGys-xukwmhdDsSaCYcMyWyeEgD4Dyg,18428
+lmnr/sdk/log.py,sha256=cZBeUoSK39LMEV-X4-eEhTWOciULRfHaKfRK8YqIM8I,1532
+lmnr/sdk/types.py,sha256=6_C2LhcbI9PwlntnSuREE0FRsBUxc3WS_yC_Y_trPBI,5052
 lmnr/sdk/utils.py,sha256=s81p6uJehgJSaLWy3sR5fTpEDH7vzn3i_UujUHChl6M,3346
 lmnr/traceloop_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/traceloop_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
 lmnr/traceloop_sdk/__init__.py,sha256=hp3q1OsFaGgaQCEanJrL38BJN32hWqCNVCSjYpndEsY,2957
 lmnr/traceloop_sdk/config/__init__.py,sha256=DliMGp2NjYAqRFLKpWQPUKjGMHRO8QsVfazBA1qENQ8,248
 lmnr/traceloop_sdk/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lmnr/traceloop_sdk/decorators/base.py,sha256=ZkvzVJfO4fOxRHwPobPL2ryW32nZjgO7cX7CXouyUnY,5621
+lmnr/traceloop_sdk/decorators/base.py,sha256=SeImd8SAeqdC8LV1bvoQUq0d_TLkOLyd5HCe7vXKhNU,5559
 lmnr/traceloop_sdk/instruments.py,sha256=oMvIASueW3GeChpjIdH-DD9aFBVB8OtHZ0HawppTrlI,942
 lmnr/traceloop_sdk/tests/__init__.py,sha256=RYnG0-8zbXL0-2Ste1mEBf5sN4d_rQjGTCgPBuaZC74,20
 lmnr/traceloop_sdk/tests/cassettes/test_association_properties/test_langchain_and_external_association_properties.yaml,sha256=26g0wRA0juicHg_XrhcE8H4vhs1lawDs0o0aLFn-I7w,3103
@@ -35,7 +36,7 @@ lmnr/traceloop_sdk/tests/test_sdk_initialization.py,sha256=fRaf6lrxFzJIN94P1Tav_
 lmnr/traceloop_sdk/tests/test_tasks.py,sha256=xlEx8BKp4yG83SCjK5WkPGfyC33JSrx4h8VyjVwGbgw,906
 lmnr/traceloop_sdk/tests/test_workflows.py,sha256=RVcfY3WAFIDZC15-aSua21aoQyYeWE7KypDyUsm-2EM,9372
 lmnr/traceloop_sdk/tracing/__init__.py,sha256=Ckq7zCM26VdJVB5tIZv0GTPyMZKyfso_KWD5yPHaqdo,66
-lmnr/traceloop_sdk/tracing/attributes.py,sha256=YBxDZ-9I9QlnehcCIUQcpeHTtv4gPU3vDtct7SqQ06Q,746
+lmnr/traceloop_sdk/tracing/attributes.py,sha256=QzfFVhbuYnT-ym4lHocQF_e5zm5Aw8r36FTxV5Qrjlk,1136
 lmnr/traceloop_sdk/tracing/content_allow_list.py,sha256=3feztm6PBWNelc8pAZUcQyEGyeSpNiVKjOaDk65l2ps,846
 lmnr/traceloop_sdk/tracing/context_manager.py,sha256=csVlB6kDmbgSPsROHwnddvGGblx55v6lJMRj0wsSMQM,304
 lmnr/traceloop_sdk/tracing/tracing.py,sha256=_HDLuyy4XgobC1ig4qz5jYbB4tWAZSfD6gbgUqwmYJU,35522
@@ -44,8 +45,8 @@ lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQv
 lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
 lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
 lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
-lmnr-0.4.23.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.23.dist-info/METADATA,sha256=BOAKiIyn49ReowUuPFpmqRYhgiQlK6CUSEEipTreYbo,10493
-lmnr-0.4.23.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lmnr-0.4.23.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.23.dist-info/RECORD,,
+lmnr-0.4.24.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.24.dist-info/METADATA,sha256=XcxnC9fKw0Hth1WXQcH_LZpT9A4TJDqWwbPre696nQU,10533
+lmnr-0.4.24.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lmnr-0.4.24.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.24.dist-info/RECORD,,