lmnr 0.4.29b1__py3-none-any.whl → 0.4.29b3__py3-none-any.whl

lmnr/sdk/evaluations.py

@@ -100,7 +100,7 @@ class Evaluation:
         data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
         executor: Any,
         evaluators: dict[str, EvaluatorFunction],
-        human_evaluators: dict[str, HumanEvaluator] = {},
+        human_evaluators: list[HumanEvaluator] = [],
         name: Optional[str] = None,
         group_id: Optional[str] = None,
         batch_size: int = DEFAULT_BATCH_SIZE,
@@ -128,11 +128,10 @@ class Evaluation:
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
-            human_evaluators (dict[str, HumanEvaluator], optional):\
-                [Beta] Dictionary from human evaluator names to instances of\
-                HumanEvaluator. For now, human evaluator only holds the queue\
-                name.
-                Defaults to an empty dictionary.
+            human_evaluators (list[HumanEvaluator], optional):\
+                [Beta] List of instances of HumanEvaluator. For now, human\
+                evaluator only holds the queue name.
+                Defaults to an empty list.
             name (Optional[str], optional): Optional name of the evaluation.\
                 Used to identify the evaluation in the group.\
                 If not provided, a random name will be generated.
@@ -174,19 +173,6 @@ class Evaluation:
                     "Keys must only contain letters, digits, hyphens,"
                     "underscores, or spaces."
                 )
-        for evaluator_name in human_evaluators or {}:
-            if not evaluator_name_regex.match(evaluator_name):
-                raise ValueError(
-                    f'Invalid human evaluator key: "{evaluator_name}". '
-                    "Keys must only contain letters, digits, hyphens,"
-                    "underscores, or spaces."
-                )
-
-        if intersection := set(evaluators.keys()) & set(human_evaluators.keys()):
-            raise ValueError(
-                "Evaluator and human evaluator names must not overlap. "
-                f"Repeated keys: {intersection}"
-            )
 
         self.is_finished = False
         self.reporter = EvaluationReporter()
@@ -281,6 +267,9 @@ class Evaluation:
                     else self.executor(datapoint.data)
                 )
                 L.set_span_output(output)
+                executor_span_id = uuid.UUID(
+                    int=executor_span.get_span_context().span_id
+                )
             target = datapoint.target
 
             # Iterate over evaluators
@@ -310,6 +299,7 @@ class Evaluation:
                 executor_output=output,
                 scores=scores,
                 trace_id=trace_id,
+                executor_span_id=executor_span_id,
             )
 
 
@@ -317,7 +307,7 @@ def evaluate(
     data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
     executor: ExecutorFunction,
     evaluators: dict[str, EvaluatorFunction],
-    human_evaluators: dict[str, HumanEvaluator] = {},
+    human_evaluators: list[HumanEvaluator] = [],
     name: Optional[str] = None,
     group_id: Optional[str] = None,
     batch_size: int = DEFAULT_BATCH_SIZE,
@@ -352,10 +342,10 @@ def evaluate(
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
-        human_evaluators (dict[str, HumanEvaluator], optional):\
-            [Beta] Dictionary from human evaluator names to instances of\
-            HumanEvaluator. For now, human evaluator only holds the queue name.
-            Defaults to an empty dictionary.
+        human_evaluators (list[HumanEvaluator], optional):\
+            [Beta] List of instances of HumanEvaluator. For now, human\
+            evaluator only holds the queue name.
+            Defaults to an empty list.
         name (Optional[str], optional): Optional name of the evaluation.\
                         Used to identify the evaluation in the group.\
                         If not provided, a random name will be generated.

lmnr/sdk/types.py

@@ -77,18 +77,17 @@ class PipelineRunError(Exception):
             return super().__str__()
 
 
-EvaluationDatapointData = dict[str, Any]
-EvaluationDatapointTarget = dict[str, Any]
-EvaluationDatapointMetadata = Optional[dict[str, Any]]
+EvaluationDatapointData = Any  # non-null, must be JSON-serializable
+EvaluationDatapointTarget = Optional[Any]  # must be JSON-serializable
+EvaluationDatapointMetadata = Optional[Any]  # must be JSON-serializable
 
 
 # EvaluationDatapoint is a single data point in the evaluation
 class Datapoint(pydantic.BaseModel):
-    # input to the executor function. Must be a dict with string keys
+    # input to the executor function.
     data: EvaluationDatapointData
     # input to the evaluator function (alongside the executor output).
-    # Must be a dict with string keys
-    target: EvaluationDatapointTarget
+    target: EvaluationDatapointTarget = pydantic.Field(default=None)
     metadata: EvaluationDatapointMetadata = pydantic.Field(default=None)
 
 
@@ -132,26 +131,29 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     scores: dict[str, Numeric]
     human_evaluators: dict[str, HumanEvaluator] = pydantic.Field(default_factory=dict)
     trace_id: uuid.UUID
+    executor_span_id: uuid.UUID
 
     # uuid is not serializable by default, so we need to convert it to a string
     def to_dict(self):
-        return {
-            "data": {
-                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
-                for k, v in self.data.items()
-            },
-            "target": {
-                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
-                for k, v in self.target.items()
-            },
-            "executorOutput": serialize(self.executor_output),
-            "scores": self.scores,
-            "traceId": str(self.trace_id),
-            "humanEvaluators": {
-                k: v.model_dump() if isinstance(v, pydantic.BaseModel) else serialize(v)
-                for k, v in self.human_evaluators.items()
-            },
-        }
+        try:
+            return {
+                "data": serialize(self.data),
+                "target": serialize(self.target),
+                "executorOutput": serialize(self.executor_output),
+                "scores": self.scores,
+                "traceId": str(self.trace_id),
+                "humanEvaluators": [
+                    (
+                        v.model_dump()
+                        if isinstance(v, pydantic.BaseModel)
+                        else serialize(v)
+                    )
+                    for v in self.human_evaluators
+                ],
+                "executorSpanId": str(self.executor_span_id),
+            }
+        except Exception as e:
+            raise ValueError(f"Error serializing EvaluationResultDatapoint: {e}")
 
 
 class SpanType(Enum):
@@ -165,7 +167,7 @@ class SpanType(Enum):
 
 class TraceType(Enum):
     DEFAULT = "DEFAULT"
-    EVENT = "EVENT"  # must not be set manually
+    EVENT = "EVENT"  # deprecated
     EVALUATION = "EVALUATION"
 
 

lmnr/sdk/utils.py

@@ -50,7 +50,7 @@ def is_iterator(o: typing.Any) -> bool:
 
 
 def serialize(obj: typing.Any) -> dict[str, typing.Any]:
-    def to_dict_inner(o: typing.Any):
+    def serialize_inner(o: typing.Any):
         if isinstance(o, (datetime.datetime, datetime.date)):
             return o.strftime("%Y-%m-%dT%H:%M:%S.%f%z")
         elif o is None:
@@ -68,17 +68,17 @@ def serialize(obj: typing.Any) -> dict[str, typing.Any]:
         elif isinstance(o, pydantic.BaseModel):
             return o.model_dump()
         elif isinstance(o, (tuple, set, frozenset)):
-            return [to_dict_inner(item) for item in o]
+            return [serialize_inner(item) for item in o]
         elif isinstance(o, list):
-            return [to_dict_inner(item) for item in o]
+            return [serialize_inner(item) for item in o]
         elif isinstance(o, dict):
-            return {to_dict_inner(k): to_dict_inner(v) for k, v in o.items()}
+            return {serialize_inner(k): serialize_inner(v) for k, v in o.items()}
         elif isinstance(o, queue.Queue):
             return type(o).__name__
 
         return str(o)
 
-    return to_dict_inner(obj)
+    return serialize_inner(obj)
 
 
 def get_input_from_func_args(

{lmnr-0.4.29b1.dist-info → lmnr-0.4.29b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.29b1
+Version: 0.4.29b3
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.4.29b1.dist-info → lmnr-0.4.29b3.dist-info}/RECORD

@@ -3,11 +3,11 @@ lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/sdk/datasets.py,sha256=w8U9E6fvetAo65Cb2CbYzlfhY8CfXAR-VysrakG6-4I,1591
 lmnr/sdk/decorators.py,sha256=ZSDaEZyjo-RUzRCltsNbe6x0t9SKl2xRQ2q4uaKvXtk,2250
-lmnr/sdk/evaluations.py,sha256=BUdsxuh3Rjk-8oj-481geW7fqTmgLcq0CuMOYbgMFx8,16807
+lmnr/sdk/evaluations.py,sha256=5Vfyp0aIjuGpqfuM3cqsaaLpcoO7z6lcOOKxnyHCNHk,16264
 lmnr/sdk/laminar.py,sha256=H87fXSWb9shcPW4AeoYwvTXJ-jSTjzm2sI1A1U1Vkg8,18780
 lmnr/sdk/log.py,sha256=cZBeUoSK39LMEV-X4-eEhTWOciULRfHaKfRK8YqIM8I,1532
-lmnr/sdk/types.py,sha256=kj-xIe3uK2WPu47RjP6xIMWnasv_SQybptkx1OVovq8,5486
-lmnr/sdk/utils.py,sha256=s81p6uJehgJSaLWy3sR5fTpEDH7vzn3i_UujUHChl6M,3346
+lmnr/sdk/types.py,sha256=CHbKYnEkiwsEU3Fcnoz4tDawrjII2RLYhP6hzc3-t_M,5593
+lmnr/sdk/utils.py,sha256=Uk8y15x-sd5tP2ERONahElLDJVEy_3dA_1_5g9A6auY,3358
 lmnr/traceloop_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/traceloop_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
 lmnr/traceloop_sdk/__init__.py,sha256=hp3q1OsFaGgaQCEanJrL38BJN32hWqCNVCSjYpndEsY,2957
@@ -45,8 +45,8 @@ lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQv
 lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
 lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
 lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
-lmnr-0.4.29b1.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.29b1.dist-info/METADATA,sha256=DrDBNTFOlwaWITf_iVv8vSrKZX2-cxMje0y69jcyceU,10690
-lmnr-0.4.29b1.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lmnr-0.4.29b1.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.29b1.dist-info/RECORD,,
+lmnr-0.4.29b3.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.29b3.dist-info/METADATA,sha256=UQ97DYAQ772h0XegYk-od_sdoOTnD_hTM-M-wX5TWLQ,10690
+lmnr-0.4.29b3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lmnr-0.4.29b3.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.29b3.dist-info/RECORD,,