lmnr 0.4.2__py3-none-any.whl → 0.4.4__py3-none-any.whl

lmnr/sdk/evaluations.py

@@ -1,7 +1,7 @@
 from typing import Union
 
+from .types import EvaluatorFunction, ExecutorFunction, EvaluationDatapoint
 from .utils import is_async
-from .types import EvaluatorFunction, ExecutorFunction, EvaluationDatapoint, Numeric
 from .laminar import Laminar as L
 import asyncio
 
@@ -95,7 +95,7 @@ class Evaluation:
         self.batch_size = batch_size
         L.initialize(project_api_key=project_api_key, base_url=base_url)
 
-    async def run(self):
+    def run(self):
         """Runs the evaluation.
 
         Creates a new evaluation if no evaluation with such name exists, or
@@ -103,61 +103,76 @@ class Evaluation:
         batches of `self.batch_size`. The executor
         function is called on each data point to get the output,
         and then evaluate it by each evaluator function.
+
+        Usage:
+        ```python
+        # in a synchronous context:
+        e.run()
+        # in an asynchronous context:
+        await e.run()
+        ```
+
         """
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            return loop.create_task(self._run())
+        else:
+            return loop.run_until_complete(self._run())
+
+    async def _run(self):
         response = L.create_evaluation(self.name)
-        batch_promises = []
 
+        # Process batches sequentially
         for i in range(0, len(self.data), self.batch_size):
             batch = (
                 self.data[i : i + self.batch_size]
                 if isinstance(self.data, list)
                 else self.data.slice(i, i + self.batch_size)
             )
-            batch_promises.append(self._evaluate_batch(batch))
+            try:
+                await self._evaluate_batch(batch)
+            except Exception as e:
+                print(f"Error evaluating batch: {e}")
 
         try:
-            await asyncio.gather(*batch_promises)
             L.update_evaluation_status(response.name, "Finished")
             print(f"Evaluation {response.id} complete")
         except Exception as e:
-            print(f"Error evaluating batch: {e}")
+            print(f"Error updating evaluation status: {e}")
 
     async def _evaluate_batch(self, batch: list[EvaluationDatapoint]):
-        results = []
-        for datapoint in batch:
-            output = (
-                await self.executor(datapoint.data)
-                if is_async(self.executor)
-                else self.executor(datapoint.data)
-            )
-            target = datapoint.target
-
-            # iterate in order of evaluators
-            scores = {}
-            for evaluator_name in self.evaluator_names:
-                evaluator = self.evaluators[evaluator_name]
-                value = (
-                    await evaluator(output, target)
-                    if is_async(evaluator)
-                    else evaluator(output, target)
-                )
+        batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
+        results = await asyncio.gather(*batch_promises)
+
+        return L.post_evaluation_results(self.name, results)
 
-                # if the evaluator returns a single number,
-                # use the evaluator name as the key
-                if isinstance(value, Numeric):
-                    scores[evaluator_name] = value
-                else:
-                    # if the evaluator returns an object,
-                    # use the object keys as the keys
-                    scores.update(value)
-
-            results.append(
-                {
-                    "executorOutput": output,
-                    "data": datapoint.data,
-                    "target": target,
-                    "scores": scores,
-                }
+    async def _evaluate_datapoint(self, datapoint):
+        output = (
+            await self.executor(datapoint.data)
+            if is_async(self.executor)
+            else self.executor(datapoint.data)
+        )
+        target = datapoint.target
+
+        # Iterate over evaluators
+        scores = {}
+        for evaluator_name in self.evaluator_names:
+            evaluator = self.evaluators[evaluator_name]
+            value = (
+                await evaluator(output, target)
+                if is_async(evaluator)
+                else evaluator(output, target)
             )
 
-        return L.post_evaluation_results(self.name, results)
+            # If evaluator returns a single number, use evaluator name as key
+            if isinstance(value, (int, float)):
+                scores[evaluator_name] = value
+            else:
+                scores.update(value)
+
+        return {
+            "executorOutput": output,
+            "data": datapoint.data,
+            "target": target,
+            "scores": scores,
+        }

lmnr/sdk/laminar.py

@@ -201,15 +201,21 @@ class Laminar:
     def event(
         cls,
         name: str,
-        value: AttributeValue,
+        value: Optional[AttributeValue] = None,
         timestamp: Optional[Union[datetime.datetime, int]] = None,
     ):
-        """Associate an event with the current span
+        """Associate an event with the current span. If event with such name never
+        existed, Laminar will create a new event and infer its type from the value.
+        If the event already exists, Laminar will append the value to the event
+        if and only if the value is of a matching type. Otherwise, the event won't
+        be recorded Supported types are string, numeric, and boolean. If the value
+        is `None`, event is considered a boolean tag with the value of `True`.
 
         Args:
             name (str): event name
-            value (AttributeValue): event value. Must be a primitive type
-                            or a sequence of values of the same primitive type
+            value (Optional[AttributeValue]): event value. Must be a primitive type.
+                            Boolean true is assumed in the backend if value is None.
+                            Defaults to None.
             timestamp (Optional[Union[datetime.datetime, int]], optional):
                             If int, must be epoch nanoseconds. If not
                             specified, relies on the underlying OpenTelemetry
@@ -220,8 +226,9 @@ class Laminar:
 
         event = {
             "lmnr.event.type": "default",
-            "lmnr.event.value": value,
         }
+        if value is not None:
+            event["lmnr.event.value"] = value
 
         current_span = get_current_span()
         if current_span == INVALID_SPAN:

{lmnr-0.4.2.dist-info → lmnr-0.4.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.2
+Version: 0.4.4
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.4.2.dist-info → lmnr-0.4.4.dist-info}/RECORD

@@ -1,13 +1,13 @@
 lmnr/__init__.py,sha256=wQwnHl662Xcz7GdSofFsEjmAK0nxioYA2Yq6Q78m4ps,194
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/sdk/decorators.py,sha256=Xs6n0TGX9LZ9i1hE_UZz4LEyd_ZAfpVGfNQh_rKwOuA,2493
-lmnr/sdk/evaluations.py,sha256=LkQApHAhR7y_rC2ovnJi8yHpdcl0-7yesdBqvOJ0BKg,6107
-lmnr/sdk/laminar.py,sha256=pMgWugFvzqYaEf1TRIjfFl_yo17GdOGGagnPBEA8nHQ,16520
+lmnr/sdk/evaluations.py,sha256=Z0j2HyXgrwlGyiT_Ql7W3e_ZWjOlNlIj9RWAKjEgkkE,6366
+lmnr/sdk/laminar.py,sha256=DQ6EvuYK0elERNDcGyKyMmHrkEJZT66-vgBmCJ8fv4Q,17077
 lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
 lmnr/sdk/types.py,sha256=gDwRSWR9A1__FGtQhVaFc6PUYQuIhubo5tpfYAajTQQ,4055
 lmnr/sdk/utils.py,sha256=ZsGJ86tq8lIbvOhSb1gJWH5K3GylO_lgX68FN6rG2nM,3358
-lmnr-0.4.2.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.2.dist-info/METADATA,sha256=92uFLKPcVAJBbWApMCY8-Mmj--ks5yXdy4Ce1jHss18,7025
-lmnr-0.4.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lmnr-0.4.2.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
-lmnr-0.4.2.dist-info/RECORD,,
+lmnr-0.4.4.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.4.dist-info/METADATA,sha256=gBmo93OeKl55jRIEJ4Fz8bvb1CTZzWVfCp3HQ8BAKHw,7025
+lmnr-0.4.4.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lmnr-0.4.4.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
+lmnr-0.4.4.dist-info/RECORD,,