lmnr 0.4.29b2__py3-none-any.whl → 0.4.29b3__py3-none-any.whl

lmnr/sdk/evaluations.py

@@ -100,7 +100,7 @@ class Evaluation:
         data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
         executor: Any,
         evaluators: dict[str, EvaluatorFunction],
-        human_evaluators: dict[str, HumanEvaluator] = {},
+        human_evaluators: list[HumanEvaluator] = [],
         name: Optional[str] = None,
         group_id: Optional[str] = None,
         batch_size: int = DEFAULT_BATCH_SIZE,
@@ -128,11 +128,10 @@ class Evaluation:
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
-            human_evaluators (dict[str, HumanEvaluator], optional):\
-                [Beta] Dictionary from human evaluator names to instances of\
-                HumanEvaluator. For now, human evaluator only holds the queue\
-                name.
-                Defaults to an empty dictionary.
+            human_evaluators (list[HumanEvaluator], optional):\
+                [Beta] List of instances of HumanEvaluator. For now, human\
+                evaluator only holds the queue name.
+                Defaults to an empty list.
             name (Optional[str], optional): Optional name of the evaluation.\
                 Used to identify the evaluation in the group.\
                 If not provided, a random name will be generated.
@@ -174,19 +173,6 @@ class Evaluation:
                     "Keys must only contain letters, digits, hyphens,"
                     "underscores, or spaces."
                 )
-        for evaluator_name in human_evaluators or {}:
-            if not evaluator_name_regex.match(evaluator_name):
-                raise ValueError(
-                    f'Invalid human evaluator key: "{evaluator_name}". '
-                    "Keys must only contain letters, digits, hyphens,"
-                    "underscores, or spaces."
-                )
-
-        if intersection := set(evaluators.keys()) & set(human_evaluators.keys()):
-            raise ValueError(
-                "Evaluator and human evaluator names must not overlap. "
-                f"Repeated keys: {intersection}"
-            )
 
         self.is_finished = False
         self.reporter = EvaluationReporter()
@@ -281,6 +267,9 @@ class Evaluation:
                     else self.executor(datapoint.data)
                 )
                 L.set_span_output(output)
+                executor_span_id = uuid.UUID(
+                    int=executor_span.get_span_context().span_id
+                )
             target = datapoint.target
 
             # Iterate over evaluators
@@ -310,6 +299,7 @@ class Evaluation:
                 executor_output=output,
                 scores=scores,
                 trace_id=trace_id,
+                executor_span_id=executor_span_id,
             )
 
 
@@ -317,7 +307,7 @@ def evaluate(
     data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
     executor: ExecutorFunction,
     evaluators: dict[str, EvaluatorFunction],
-    human_evaluators: dict[str, HumanEvaluator] = {},
+    human_evaluators: list[HumanEvaluator] = [],
     name: Optional[str] = None,
     group_id: Optional[str] = None,
     batch_size: int = DEFAULT_BATCH_SIZE,
@@ -352,10 +342,10 @@ def evaluate(
                 If the score is a single number, it will be named after the\
                 evaluator function. Evaluator function names must contain only\
                 letters, digits, hyphens, underscores, or spaces.
-        human_evaluators (dict[str, HumanEvaluator], optional):\
-            [Beta] Dictionary from human evaluator names to instances of\
-            HumanEvaluator. For now, human evaluator only holds the queue name.
-            Defaults to an empty dictionary.
+        human_evaluators (list[HumanEvaluator], optional):\
+            [Beta] List of instances of HumanEvaluator. For now, human\
+            evaluator only holds the queue name.
+            Defaults to an empty list.
         name (Optional[str], optional): Optional name of the evaluation.\
                         Used to identify the evaluation in the group.\
                         If not provided, a random name will be generated.

lmnr/sdk/types.py

@@ -131,6 +131,7 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
     scores: dict[str, Numeric]
     human_evaluators: dict[str, HumanEvaluator] = pydantic.Field(default_factory=dict)
     trace_id: uuid.UUID
+    executor_span_id: uuid.UUID
 
     # uuid is not serializable by default, so we need to convert it to a string
     def to_dict(self):
@@ -141,14 +142,15 @@ class EvaluationResultDatapoint(pydantic.BaseModel):
                 "executorOutput": serialize(self.executor_output),
                 "scores": self.scores,
                 "traceId": str(self.trace_id),
-                "humanEvaluators": {
-                    k: (
+                "humanEvaluators": [
+                    (
                         v.model_dump()
                         if isinstance(v, pydantic.BaseModel)
                         else serialize(v)
                     )
-                    for k, v in self.human_evaluators.items()
-                },
+                    for v in self.human_evaluators
+                ],
+                "executorSpanId": str(self.executor_span_id),
             }
         except Exception as e:
             raise ValueError(f"Error serializing EvaluationResultDatapoint: {e}")

{lmnr-0.4.29b2.dist-info → lmnr-0.4.29b3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.29b2
+Version: 0.4.29b3
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.4.29b2.dist-info → lmnr-0.4.29b3.dist-info}/RECORD

@@ -3,10 +3,10 @@ lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/sdk/datasets.py,sha256=w8U9E6fvetAo65Cb2CbYzlfhY8CfXAR-VysrakG6-4I,1591
 lmnr/sdk/decorators.py,sha256=ZSDaEZyjo-RUzRCltsNbe6x0t9SKl2xRQ2q4uaKvXtk,2250
-lmnr/sdk/evaluations.py,sha256=BUdsxuh3Rjk-8oj-481geW7fqTmgLcq0CuMOYbgMFx8,16807
+lmnr/sdk/evaluations.py,sha256=5Vfyp0aIjuGpqfuM3cqsaaLpcoO7z6lcOOKxnyHCNHk,16264
 lmnr/sdk/laminar.py,sha256=H87fXSWb9shcPW4AeoYwvTXJ-jSTjzm2sI1A1U1Vkg8,18780
 lmnr/sdk/log.py,sha256=cZBeUoSK39LMEV-X4-eEhTWOciULRfHaKfRK8YqIM8I,1532
-lmnr/sdk/types.py,sha256=SyvJ9WMdsfW0Blqp579hfTC4A34YfmYxxY5QKkS4eVs,5513
+lmnr/sdk/types.py,sha256=CHbKYnEkiwsEU3Fcnoz4tDawrjII2RLYhP6hzc3-t_M,5593
 lmnr/sdk/utils.py,sha256=Uk8y15x-sd5tP2ERONahElLDJVEy_3dA_1_5g9A6auY,3358
 lmnr/traceloop_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/traceloop_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
@@ -45,8 +45,8 @@ lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQv
 lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
 lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
 lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
-lmnr-0.4.29b2.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.29b2.dist-info/METADATA,sha256=fWqZcMMmcJ68dGr-5oWyLMT2ac_lBR9vWfolKzqQjIw,10690
-lmnr-0.4.29b2.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lmnr-0.4.29b2.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.29b2.dist-info/RECORD,,
+lmnr-0.4.29b3.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.29b3.dist-info/METADATA,sha256=UQ97DYAQ772h0XegYk-od_sdoOTnD_hTM-M-wX5TWLQ,10690
+lmnr-0.4.29b3.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lmnr-0.4.29b3.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.29b3.dist-info/RECORD,,