lmnr 0.4.2__tar.gz → 0.4.4__tar.gz

{lmnr-0.4.2 → lmnr-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.2
+Version: 0.4.4
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.4.2 → lmnr-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lmnr"
-version = "0.4.2"
+version = "0.4.4"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -11,7 +11,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.2"
+version = "0.4.4"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"

{lmnr-0.4.2 → lmnr-0.4.4}/src/lmnr/sdk/evaluations.py

@@ -1,7 +1,7 @@
 from typing import Union
 
+from .types import EvaluatorFunction, ExecutorFunction, EvaluationDatapoint
 from .utils import is_async
-from .types import EvaluatorFunction, ExecutorFunction, EvaluationDatapoint, Numeric
 from .laminar import Laminar as L
 import asyncio
 
@@ -95,7 +95,7 @@ class Evaluation:
         self.batch_size = batch_size
         L.initialize(project_api_key=project_api_key, base_url=base_url)
 
-    async def run(self):
+    def run(self):
         """Runs the evaluation.
 
         Creates a new evaluation if no evaluation with such name exists, or
@@ -103,61 +103,76 @@ class Evaluation:
         batches of `self.batch_size`. The executor
         function is called on each data point to get the output,
         and then evaluate it by each evaluator function.
+
+        Usage:
+        ```python
+        # in a synchronous context:
+        e.run()
+        # in an asynchronous context:
+        await e.run()
+        ```
+
         """
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            return loop.create_task(self._run())
+        else:
+            return loop.run_until_complete(self._run())
+
+    async def _run(self):
         response = L.create_evaluation(self.name)
-        batch_promises = []
 
+        # Process batches sequentially
         for i in range(0, len(self.data), self.batch_size):
             batch = (
                 self.data[i : i + self.batch_size]
                 if isinstance(self.data, list)
                 else self.data.slice(i, i + self.batch_size)
             )
-            batch_promises.append(self._evaluate_batch(batch))
+            try:
+                await self._evaluate_batch(batch)
+            except Exception as e:
+                print(f"Error evaluating batch: {e}")
 
         try:
-            await asyncio.gather(*batch_promises)
             L.update_evaluation_status(response.name, "Finished")
             print(f"Evaluation {response.id} complete")
         except Exception as e:
-            print(f"Error evaluating batch: {e}")
+            print(f"Error updating evaluation status: {e}")
 
     async def _evaluate_batch(self, batch: list[EvaluationDatapoint]):
-        results = []
-        for datapoint in batch:
-            output = (
-                await self.executor(datapoint.data)
-                if is_async(self.executor)
-                else self.executor(datapoint.data)
-            )
-            target = datapoint.target
-
-            # iterate in order of evaluators
-            scores = {}
-            for evaluator_name in self.evaluator_names:
-                evaluator = self.evaluators[evaluator_name]
-                value = (
-                    await evaluator(output, target)
-                    if is_async(evaluator)
-                    else evaluator(output, target)
-                )
+        batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
+        results = await asyncio.gather(*batch_promises)
+
+        return L.post_evaluation_results(self.name, results)
 
-                # if the evaluator returns a single number,
-                # use the evaluator name as the key
-                if isinstance(value, Numeric):
-                    scores[evaluator_name] = value
-                else:
-                    # if the evaluator returns an object,
-                    # use the object keys as the keys
-                    scores.update(value)
-
-            results.append(
-                {
-                    "executorOutput": output,
-                    "data": datapoint.data,
-                    "target": target,
-                    "scores": scores,
-                }
+    async def _evaluate_datapoint(self, datapoint):
+        output = (
+            await self.executor(datapoint.data)
+            if is_async(self.executor)
+            else self.executor(datapoint.data)
+        )
+        target = datapoint.target
+
+        # Iterate over evaluators
+        scores = {}
+        for evaluator_name in self.evaluator_names:
+            evaluator = self.evaluators[evaluator_name]
+            value = (
+                await evaluator(output, target)
+                if is_async(evaluator)
+                else evaluator(output, target)
             )
 
-        return L.post_evaluation_results(self.name, results)
+            # If evaluator returns a single number, use evaluator name as key
+            if isinstance(value, (int, float)):
+                scores[evaluator_name] = value
+            else:
+                scores.update(value)
+
+        return {
+            "executorOutput": output,
+            "data": datapoint.data,
+            "target": target,
+            "scores": scores,
+        }

{lmnr-0.4.2 → lmnr-0.4.4}/src/lmnr/sdk/laminar.py

@@ -201,15 +201,21 @@ class Laminar:
     def event(
         cls,
         name: str,
-        value: AttributeValue,
+        value: Optional[AttributeValue] = None,
         timestamp: Optional[Union[datetime.datetime, int]] = None,
     ):
-        """Associate an event with the current span
+        """Associate an event with the current span. If event with such name never
+        existed, Laminar will create a new event and infer its type from the value.
+        If the event already exists, Laminar will append the value to the event
+        if and only if the value is of a matching type. Otherwise, the event won't
+        be recorded Supported types are string, numeric, and boolean. If the value
+        is `None`, event is considered a boolean tag with the value of `True`.
 
         Args:
             name (str): event name
-            value (AttributeValue): event value. Must be a primitive type
-                            or a sequence of values of the same primitive type
+            value (Optional[AttributeValue]): event value. Must be a primitive type.
+                            Boolean true is assumed in the backend if value is None.
+                            Defaults to None.
             timestamp (Optional[Union[datetime.datetime, int]], optional):
                             If int, must be epoch nanoseconds. If not
                             specified, relies on the underlying OpenTelemetry
@@ -220,8 +226,9 @@ class Laminar:
 
         event = {
             "lmnr.event.type": "default",
-            "lmnr.event.value": value,
         }
+        if value is not None:
+            event["lmnr.event.value"] = value
 
         current_span = get_current_span()
         if current_span == INVALID_SPAN: