lmnr 0.4.2__py3-none-any.whl → 0.4.3__py3-none-any.whl

lmnr/sdk/evaluations.py

@@ -105,59 +105,58 @@ class Evaluation:
         and then evaluate it by each evaluator function.
         """
         response = L.create_evaluation(self.name)
-        batch_promises = []
 
+        # Process batches sequentially
         for i in range(0, len(self.data), self.batch_size):
             batch = (
                 self.data[i : i + self.batch_size]
                 if isinstance(self.data, list)
                 else self.data.slice(i, i + self.batch_size)
             )
-            batch_promises.append(self._evaluate_batch(batch))
+            try:
+                await self._evaluate_batch(batch)
+            except Exception as e:
+                print(f"Error evaluating batch: {e}")
 
         try:
-            await asyncio.gather(*batch_promises)
             L.update_evaluation_status(response.name, "Finished")
             print(f"Evaluation {response.id} complete")
         except Exception as e:
-            print(f"Error evaluating batch: {e}")
+            print(f"Error updating evaluation status: {e}")
 
     async def _evaluate_batch(self, batch: list[EvaluationDatapoint]):
-        results = []
-        for datapoint in batch:
-            output = (
-                await self.executor(datapoint.data)
-                if is_async(self.executor)
-                else self.executor(datapoint.data)
-            )
-            target = datapoint.target
-
-            # iterate in order of evaluators
-            scores = {}
-            for evaluator_name in self.evaluator_names:
-                evaluator = self.evaluators[evaluator_name]
-                value = (
-                    await evaluator(output, target)
-                    if is_async(evaluator)
-                    else evaluator(output, target)
-                )
+        batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
+        results = await asyncio.gather(*batch_promises)
+
+        return L.post_evaluation_results(self.name, results)
 
-                # if the evaluator returns a single number,
-                # use the evaluator name as the key
-                if isinstance(value, Numeric):
-                    scores[evaluator_name] = value
-                else:
-                    # if the evaluator returns an object,
-                    # use the object keys as the keys
-                    scores.update(value)
-
-            results.append(
-                {
-                    "executorOutput": output,
-                    "data": datapoint.data,
-                    "target": target,
-                    "scores": scores,
-                }
+    async def _evaluate_datapoint(self, datapoint):
+        output = (
+            await self.executor(datapoint.data)
+            if asyncio.iscoroutinefunction(self.executor)
+            else self.executor(datapoint.data)
+        )
+        target = datapoint.target
+
+        # Iterate over evaluators
+        scores = {}
+        for evaluator_name in self.evaluator_names:
+            evaluator = self.evaluators[evaluator_name]
+            value = (
+                await evaluator(output, target)
+                if asyncio.iscoroutinefunction(evaluator)
+                else evaluator(output, target)
             )
 
-        return L.post_evaluation_results(self.name, results)
+            # If evaluator returns a single number, use evaluator name as key
+            if isinstance(value, (int, float)):
+                scores[evaluator_name] = value
+            else:
+                scores.update(value)
+
+        return {
+            "executorOutput": output,
+            "data": datapoint.data,
+            "target": target,
+            "scores": scores,
+        }

{lmnr-0.4.2.dist-info → lmnr-0.4.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.2
+Version: 0.4.3
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.4.2.dist-info → lmnr-0.4.3.dist-info}/RECORD

@@ -1,13 +1,13 @@
 lmnr/__init__.py,sha256=wQwnHl662Xcz7GdSofFsEjmAK0nxioYA2Yq6Q78m4ps,194
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/sdk/decorators.py,sha256=Xs6n0TGX9LZ9i1hE_UZz4LEyd_ZAfpVGfNQh_rKwOuA,2493
-lmnr/sdk/evaluations.py,sha256=LkQApHAhR7y_rC2ovnJi8yHpdcl0-7yesdBqvOJ0BKg,6107
+lmnr/sdk/evaluations.py,sha256=jdXWRJyr17iAWva8Vyvn8yT-rZAoQ8Xz0QQ_zzfPwvY,6044
 lmnr/sdk/laminar.py,sha256=pMgWugFvzqYaEf1TRIjfFl_yo17GdOGGagnPBEA8nHQ,16520
 lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
 lmnr/sdk/types.py,sha256=gDwRSWR9A1__FGtQhVaFc6PUYQuIhubo5tpfYAajTQQ,4055
 lmnr/sdk/utils.py,sha256=ZsGJ86tq8lIbvOhSb1gJWH5K3GylO_lgX68FN6rG2nM,3358
-lmnr-0.4.2.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.2.dist-info/METADATA,sha256=92uFLKPcVAJBbWApMCY8-Mmj--ks5yXdy4Ce1jHss18,7025
-lmnr-0.4.2.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lmnr-0.4.2.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
-lmnr-0.4.2.dist-info/RECORD,,
+lmnr-0.4.3.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.3.dist-info/METADATA,sha256=woC24QpLILvRBtLdwCktKJydeiPXEOH8LBb7CSAnw4I,7025
+lmnr-0.4.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lmnr-0.4.3.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
+lmnr-0.4.3.dist-info/RECORD,,