lmnr 0.4.28__tar.gz → 0.4.29__tar.gz

{lmnr-0.4.28 → lmnr-0.4.29}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.28
+Version: 0.4.29
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -39,6 +39,7 @@ Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.33.5)
 Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.33.5)
 Requires-Dist: opentelemetry-instrumentation-replicate (>=0.33.5)
 Requires-Dist: opentelemetry-instrumentation-requests (>=0.48b0,<0.49)
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.33.5)
 Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.48b0,<0.49)
 Requires-Dist: opentelemetry-instrumentation-threading (>=0.48b0,<0.49)
 Requires-Dist: opentelemetry-instrumentation-together (>=0.33.5)

{lmnr-0.4.28 → lmnr-0.4.29}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lmnr"
-version = "0.4.28"
+version = "0.4.29"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -11,7 +11,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.28"
+version = "0.4.29"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"
@@ -35,30 +35,31 @@ opentelemetry-semantic-conventions-ai = "0.4.2"
 tenacity = ">=8.0"
 jinja2 = "~=3.0"
 deprecated = "~=1.0"
-opentelemetry-instrumentation-mistralai = ">=0.33.5"
-opentelemetry-instrumentation-openai = ">=0.33.5"
-opentelemetry-instrumentation-ollama = ">=0.33.5"
+opentelemetry-instrumentation-alephalpha = ">=0.33.5"
 opentelemetry-instrumentation-anthropic = ">=0.33.5"
+opentelemetry-instrumentation-bedrock = ">=0.33.5"
+opentelemetry-instrumentation-chromadb = ">=0.33.5"
 opentelemetry-instrumentation-cohere = ">=0.33.5"
 opentelemetry-instrumentation-google-generativeai = ">=0.33.5"
-opentelemetry-instrumentation-pinecone = ">=0.33.5"
-opentelemetry-instrumentation-qdrant = ">=0.33.5"
-opentelemetry-instrumentation-langchain = ">=0.33.5"
+opentelemetry-instrumentation-groq = ">=0.33.5"
+opentelemetry-instrumentation-haystack = ">=0.33.5"
 opentelemetry-instrumentation-lancedb = ">=0.33.5"
-opentelemetry-instrumentation-chromadb = ">=0.33.5"
-opentelemetry-instrumentation-transformers = ">=0.33.5"
-opentelemetry-instrumentation-together = ">=0.33.5"
+opentelemetry-instrumentation-langchain = ">=0.33.5"
 opentelemetry-instrumentation-llamaindex = ">=0.33.5"
+opentelemetry-instrumentation-marqo = ">=0.33.5"
 opentelemetry-instrumentation-milvus = ">=0.33.5"
-opentelemetry-instrumentation-haystack = ">=0.33.5"
-opentelemetry-instrumentation-bedrock = ">=0.33.5"
+opentelemetry-instrumentation-mistralai = ">=0.33.5"
+opentelemetry-instrumentation-ollama = ">=0.33.5"
+opentelemetry-instrumentation-openai = ">=0.33.5"
+opentelemetry-instrumentation-pinecone = ">=0.33.5"
+opentelemetry-instrumentation-qdrant = ">=0.33.5"
 opentelemetry-instrumentation-replicate = ">=0.33.5"
+opentelemetry-instrumentation-sagemaker = ">=0.33.5"
+opentelemetry-instrumentation-together = ">=0.33.5"
+opentelemetry-instrumentation-transformers = ">=0.33.5"
 opentelemetry-instrumentation-vertexai = ">=0.33.5"
 opentelemetry-instrumentation-watsonx = ">=0.33.5"
 opentelemetry-instrumentation-weaviate = ">=0.33.5"
-opentelemetry-instrumentation-alephalpha = ">=0.33.5"
-opentelemetry-instrumentation-marqo = ">=0.33.5"
-opentelemetry-instrumentation-groq = ">=0.33.5"
 tqdm = "~=4.0"
 argparse = "~=1.0"
 

{lmnr-0.4.28 → lmnr-0.4.29}/src/lmnr/__init__.py

@@ -1,7 +1,14 @@
 from .sdk.datasets import EvaluationDataset, LaminarDataset
 from .sdk.evaluations import evaluate
 from .sdk.laminar import Laminar
-from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput
+from .sdk.types import (
+    ChatMessage,
+    HumanEvaluator,
+    NodeInput,
+    PipelineRunError,
+    PipelineRunResponse,
+)
 from .sdk.decorators import observe
 from .traceloop_sdk import Instruments
 from .traceloop_sdk.tracing.attributes import Attributes
+from opentelemetry.trace import use_span

{lmnr-0.4.28 → lmnr-0.4.29}/src/lmnr/sdk/evaluations.py

@@ -18,6 +18,7 @@ from .types import (
     EvaluationResultDatapoint,
     EvaluatorFunction,
     ExecutorFunction,
+    HumanEvaluator,
     Numeric,
     NumericTypes,
     SpanType,
@@ -99,6 +100,7 @@ class Evaluation:
         data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
         executor: Any,
         evaluators: dict[str, EvaluatorFunction],
+        human_evaluators: list[HumanEvaluator] = [],
         name: Optional[str] = None,
         group_id: Optional[str] = None,
         batch_size: int = DEFAULT_BATCH_SIZE,
@@ -126,6 +128,10 @@ class Evaluation:
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
+            human_evaluators (list[HumanEvaluator], optional):\
+                [Beta] List of instances of HumanEvaluator. For now, human\
+                evaluator only holds the queue name.
+                Defaults to an empty list.
             name (Optional[str], optional): Optional name of the evaluation.\
                 Used to identify the evaluation in the group.\
                 If not provided, a random name will be generated.
@@ -159,9 +165,9 @@ class Evaluation:
         if not evaluators:
             raise ValueError("No evaluators provided")
 
-        # TODO: Compile regex once and then reuse it
+        evaluator_name_regex = re.compile(r"^[\w\s-]+$")
         for evaluator_name in evaluators:
-            if not re.match(r"^[\w\s-]+$", evaluator_name):
+            if not evaluator_name_regex.match(evaluator_name):
                 raise ValueError(
                     f'Invalid evaluator key: "{evaluator_name}". '
                     "Keys must only contain letters, digits, hyphens,"
@@ -183,6 +189,7 @@ class Evaluation:
         self.name = name
         self.batch_size = batch_size
         self._logger = get_default_logger(self.__class__.__name__)
+        self.human_evaluators = human_evaluators
         L.initialize(
             project_api_key=project_api_key,
             base_url=base_url,
@@ -202,9 +209,7 @@ class Evaluation:
             return loop.run_until_complete(self._run())
 
     async def _run(self) -> None:
-        self.reporter.start(
-            len(self.data),
-        )
+        self.reporter.start(len(self.data))
 
         try:
             result_datapoints = await self._evaluate_in_batches()
@@ -212,13 +217,19 @@ class Evaluation:
             self.reporter.stopWithError(e)
             self.is_finished = True
             return
-        else:
-            evaluation = L.create_evaluation(
-                data=result_datapoints, group_id=self.group_id, name=self.name
-            )
-            average_scores = get_average_scores(result_datapoints)
-            self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
-            self.is_finished = True
+
+        # For now add all human evaluators to all result datapoints
+        # In the future, we will add ways to specify which human evaluators
+        # to add to which result datapoints, e.g. sample some randomly
+        for result_datapoint in result_datapoints:
+            result_datapoint.human_evaluators = self.human_evaluators or {}
+
+        evaluation = L.create_evaluation(
+            data=result_datapoints, group_id=self.group_id, name=self.name
+        )
+        average_scores = get_average_scores(result_datapoints)
+        self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
+        self.is_finished = True
 
     async def _evaluate_in_batches(self) -> list[EvaluationResultDatapoint]:
         result_datapoints = []
@@ -256,6 +267,9 @@ class Evaluation:
                     else self.executor(datapoint.data)
                 )
                 L.set_span_output(output)
+                executor_span_id = uuid.UUID(
+                    int=executor_span.get_span_context().span_id
+                )
             target = datapoint.target
 
             # Iterate over evaluators
@@ -285,6 +299,7 @@ class Evaluation:
                 executor_output=output,
                 scores=scores,
                 trace_id=trace_id,
+                executor_span_id=executor_span_id,
             )
 
 
@@ -292,6 +307,7 @@ def evaluate(
     data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
     executor: ExecutorFunction,
     evaluators: dict[str, EvaluatorFunction],
+    human_evaluators: list[HumanEvaluator] = [],
     name: Optional[str] = None,
     group_id: Optional[str] = None,
     batch_size: int = DEFAULT_BATCH_SIZE,
@@ -326,6 +342,10 @@ def evaluate(
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
+        human_evaluators (list[HumanEvaluator], optional):\
+            [Beta] List of instances of HumanEvaluator. For now, human\
+            evaluator only holds the queue name.
+            Defaults to an empty list.
         name (Optional[str], optional): Optional name of the evaluation.\
                         Used to identify the evaluation in the group.\
                         If not provided, a random name will be generated.
@@ -359,6 +379,7 @@ def evaluate(
         executor=executor,
         evaluators=evaluators,
         group_id=group_id,
+        human_evaluators=human_evaluators,
         name=name,
         batch_size=batch_size,
         project_api_key=project_api_key,

{lmnr-0.4.28 → lmnr-0.4.29}/src/lmnr/sdk/laminar.py

@@ -1,3 +1,4 @@
+from contextvars import Context
 import re
 from lmnr.traceloop_sdk.instruments import Instruments
 from opentelemetry import context
@@ -294,6 +295,7 @@ class Laminar:
         name: str,
         input: Any = None,
         span_type: Union[Literal["DEFAULT"], Literal["LLM"]] = "DEFAULT",
+        context: Optional[Context] = None,
     ):
         """Start a new span as the current span. Useful for manual
         instrumentation. If `span_type` is set to `"LLM"`, you should report
@@ -314,10 +316,12 @@ class Laminar:
             span_type (Union[Literal["DEFAULT"], Literal["LLM"]], optional):\
                 type of the span. If you use `"LLM"`, you should report usage\
                 and response attributes manually. Defaults to "DEFAULT".
+            context (Optional[Context], optional): raw OpenTelemetry context\
+                to attach the span to. Defaults to None.
         """
         with get_tracer() as tracer:
             span_path = get_span_path(name)
-            ctx = set_value("span_path", span_path)
+            ctx = set_value("span_path", span_path, context)
             ctx_token = attach(ctx)
             with tracer.start_as_current_span(
                 name,
@@ -338,6 +342,67 @@ class Laminar:
             except Exception:
                 pass
 
+    @classmethod
+    def start_span(
+        cls,
+        name: str,
+        input: Any = None,
+        span_type: Union[Literal["DEFAULT"], Literal["LLM"]] = "DEFAULT",
+        context: Optional[Context] = None,
+    ):
+        """Start a new span. Useful for manual instrumentation.
+        If `span_type` is set to `"LLM"`, you should report usage and response
+        attributes manually. See `Laminar.set_span_attributes` for more
+        information.
+
+        Usage example:
+        ```python
+        from src.lmnr import Laminar, use_span
+        def foo(span):
+            with use_span(span):
+                with Laminar.start_as_current_span("foo_inner"):
+                    some_function()
+        
+        def bar():
+            with use_span(span):
+                openai_client.chat.completions.create()
+        
+        span = Laminar.start_span("outer")
+        foo(span)
+        bar(span)
+        # IMPORTANT: End the span manually
+        span.end()
+        
+        # Results in:
+        # | outer
+        # |   | foo
+        # |   |   | foo_inner
+        # |   | bar
+        # |   |   | openai.chat
+        ```
+
+        Args:
+            name (str): name of the span
+            input (Any, optional): input to the span. Will be sent as an\
+                attribute, so must be json serializable. Defaults to None.
+            span_type (Union[Literal["DEFAULT"], Literal["LLM"]], optional):\
+                type of the span. If you use `"LLM"`, you should report usage\
+                and response attributes manually. Defaults to "DEFAULT".
+            context (Optional[Context], optional): raw OpenTelemetry context\
+                to attach the span to. Defaults to None.
+        """
+        with get_tracer() as tracer:
+            span_path = get_span_path(name)
+            ctx = set_value("span_path", span_path, context)
+            span = tracer.start_span(name, context=ctx)
+            if input is not None:
+                span.set_attribute(
+                    SPAN_INPUT,
+                    json_dumps(input),
+                )
+            span.set_attribute(SPAN_TYPE, span_type)
+            return span
+
     @classmethod
     def set_span_output(cls, output: Any = None):
         """Set the output of the current span. Useful for manual

{lmnr-0.4.28 → lmnr-0.4.29}/src/lmnr/sdk/types.py

@@ -77,18 +77,17 @@ class PipelineRunError(Exception):
             return super().__str__()
 
 
-EvaluationDatapointData = dict[str, Any]
-EvaluationDatapointTarget = dict[str, Any]
-EvaluationDatapointMetadata = Optional[dict[str, Any]]
+EvaluationDatapointData = Any  # non-null, must be JSON-serializable
+EvaluationDatapointTarget = Optional[Any]  # must be JSON-serializable
+EvaluationDatapointMetadata = Optional[Any]  # must be JSON-serializable
 
 
 # EvaluationDatapoint is a single data point in the evaluation
 class Datapoint(pydantic.BaseModel):
-    # input to the executor function. Must be a dict with string keys
+    # input to the executor function.
     data: EvaluationDatapointData
     # input to the evaluator function (alongside the executor output).
-    # Must be a dict with string keys
-    target: EvaluationDatapointTarget
+    target: EvaluationDatapointTarget = pydantic.Field(default=None)
     metadata: EvaluationDatapointMetadata = pydantic.Field(default=None)
 
 
@@ -110,6 +109,13 @@ EvaluatorFunction = Callable[
 ]
 
 
+class HumanEvaluator(pydantic.BaseModel):
+    queueName: str
+
+    def __init__(self, queue_name: str):
+        super().__init__(queueName=queue_name)
+
+
 class CreateEvaluationResponse(pydantic.BaseModel):
     id: uuid.UUID
     createdAt: datetime.datetime
@@ -123,23 +129,31 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     target: EvaluationDatapointTarget
     executor_output: ExecutorFunctionReturnType
     scores: dict[str, Numeric]
+    human_evaluators: list[HumanEvaluator] = pydantic.Field(default_factory=list)
     trace_id: uuid.UUID
+    executor_span_id: uuid.UUID
 
     # uuid is not serializable by default, so we need to convert it to a string
     def to_dict(self):
-        return {
-            "data": {
-                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
-                for k, v in self.data.items()
-            },
-            "target": {
-                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
-                for k, v in self.target.items()
-            },
-            "executorOutput": serialize(self.executor_output),
-            "scores": self.scores,
-            "traceId": str(self.trace_id),
-        }
+        try:
+            return {
+                "data": serialize(self.data),
+                "target": serialize(self.target),
+                "executorOutput": serialize(self.executor_output),
+                "scores": self.scores,
+                "traceId": str(self.trace_id),
+                "humanEvaluators": [
+                    (
+                        v.model_dump()
+                        if isinstance(v, pydantic.BaseModel)
+                        else serialize(v)
+                    )
+                    for v in self.human_evaluators
+                ],
+                "executorSpanId": str(self.executor_span_id),
+            }
+        except Exception as e:
+            raise ValueError(f"Error serializing EvaluationResultDatapoint: {e}")
 
 
 class SpanType(Enum):
@@ -153,7 +167,7 @@ class SpanType(Enum):
 
 class TraceType(Enum):
     DEFAULT = "DEFAULT"
-    EVENT = "EVENT"  # must not be set manually
+    EVENT = "EVENT"  # deprecated
     EVALUATION = "EVALUATION"
 
 

{lmnr-0.4.28 → lmnr-0.4.29}/src/lmnr/sdk/utils.py

@@ -50,7 +50,7 @@ def is_iterator(o: typing.Any) -> bool:
 
 
 def serialize(obj: typing.Any) -> dict[str, typing.Any]:
-    def to_dict_inner(o: typing.Any):
+    def serialize_inner(o: typing.Any):
         if isinstance(o, (datetime.datetime, datetime.date)):
             return o.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
         elif o is None:
@@ -68,17 +68,17 @@ def serialize(obj: typing.Any) -> dict[str, typing.Any]:
         elif isinstance(o, pydantic.BaseModel):
             return o.model_dump()
         elif isinstance(o, (tuple, set, frozenset)):
-            return [to_dict_inner(item) for item in o]
+            return [serialize_inner(item) for item in o]
         elif isinstance(o, list):
-            return [to_dict_inner(item) for item in o]
+            return [serialize_inner(item) for item in o]
         elif isinstance(o, dict):
-            return {to_dict_inner(k): to_dict_inner(v) for k, v in o.items()}
+            return {serialize_inner(k): serialize_inner(v) for k, v in o.items()}
         elif isinstance(o, queue.Queue):
             return type(o).__name__
 
         return str(o)
 
-    return to_dict_inner(obj)
+    return serialize_inner(obj)
 
 
 def get_input_from_func_args(

{lmnr-0.4.28 → lmnr-0.4.29}/src/lmnr/traceloop_sdk/instruments.py

@@ -4,27 +4,31 @@ from enum import Enum
 class Instruments(Enum):
     # The list of libraries which will be autoinstrumented
     # if no specific instruments are provided to initialize()
-    OPENAI = "openai"
+    ALEPHALPHA = "alephalpha"
     ANTHROPIC = "anthropic"
-    COHERE = "cohere"
-    PINECONE = "pinecone"
+    BEDROCK = "bedrock"
     CHROMA = "chroma"
+    COHERE = "cohere"
     GOOGLE_GENERATIVEAI = "google_generativeai"
+    GROQ = "groq"
+    HAYSTACK = "haystack"
+    LANCEDB = "lancedb"
     LANGCHAIN = "langchain"
-    MISTRAL = "mistral"
-    OLLAMA = "ollama"
     LLAMA_INDEX = "llama_index"
+    MARQO = "marqo"
     MILVUS = "milvus"
-    TRANSFORMERS = "transformers"
-    TOGETHER = "together"
-    BEDROCK = "bedrock"
+    MISTRAL = "mistral"
+    OLLAMA = "ollama"
+    OPENAI = "openai"
+    PINECONE = "pinecone"
+    QDRANT = "qdrant"
     REPLICATE = "replicate"
+    SAGEMAKER = "sagemaker"
+    TOGETHER = "together"
+    TRANSFORMERS = "transformers"
     VERTEXAI = "vertexai"
     WATSONX = "watsonx"
     WEAVIATE = "weaviate"
-    ALEPHALPHA = "alephalpha"
-    MARQO = "marqo"
-    LANCEDB = "lancedb"
 
     # The following libraries will not be autoinstrumented unless
     # specified explicitly in the initialize() call.