lmnr 0.4.12b1__tar.gz → 0.4.12b3__tar.gz

{lmnr-0.4.12b1 → lmnr-0.4.12b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.12b1
+Version: 0.4.12b3
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai

{lmnr-0.4.12b1 → lmnr-0.4.12b3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lmnr"
-version = "0.4.12b1"
+version = "0.4.12b3"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -11,7 +11,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.12b1"
+version = "0.4.12b3"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"

{lmnr-0.4.12b1 → lmnr-0.4.12b3}/src/lmnr/sdk/evaluations.py

@@ -7,7 +7,7 @@ from typing import Any, Awaitable, Optional, Union
 from tqdm import tqdm
 
 from .laminar import Laminar as L
-from .types import CreateEvaluationResponse, Datapoint, EvaluationResultDatapoint, Numeric
+from .types import CreateEvaluationResponse, Datapoint, EvaluationResultDatapoint, Numeric, NumericTypes
 from .utils import is_async
 
 DEFAULT_BATCH_SIZE = 5
@@ -76,10 +76,10 @@ class EvaluationDataset(ABC):
 class Evaluation:
     def __init__(
         self,
-        name: str,
         data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
         executor: Any,
         evaluators: list[Any],
+        name: Optional[str] = None,
         batch_size: int = DEFAULT_BATCH_SIZE,
         project_api_key: Optional[str] = None,
         base_url: Optional[str] = None,
@@ -89,7 +89,6 @@ class Evaluation:
         Initializes an instance of the Evaluations class.
 
         Parameters:
-            name (str): The name of the evaluation.
             data (Union[List[Union[EvaluationDatapoint, dict]], EvaluationDataset]): List of data points to evaluate or an evaluation dataset.
                             `data` is the input to the executor function,
                             `target` is the input to the evaluator function.
@@ -104,6 +103,8 @@ class Evaluation:
                 evaluator function. If the function is anonymous, it will be
                 named `evaluator_${index}`, where index is the index of the
                 evaluator function in the list starting from 1.
+            name (Optional[str], optional): The name of the evaluation.
+                            It will be auto-generated if not provided.
             batch_size (int, optional): The batch size for evaluation.
                             Defaults to DEFAULT_BATCH_SIZE.
             project_api_key (Optional[str], optional): The project API key.
@@ -240,7 +241,7 @@ class Evaluation:
             )
 
             # If evaluator returns a single number, use evaluator name as key
-            if isinstance(value, Numeric):
+            if isinstance(value, NumericTypes):
                 scores[evaluator_name] = value
             else:
                 scores.update(value)
@@ -254,33 +255,57 @@ class Evaluation:
 
 
 def evaluate(
-    name: str,
     data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
     executor: Any,
     evaluators: list[Any],
+    name: Optional[str] = None,
     batch_size: int = DEFAULT_BATCH_SIZE,
     project_api_key: Optional[str] = None,
     base_url: Optional[str] = None,
     http_port: Optional[int] = None,
 ) -> Optional[Awaitable[None]]:
     """
-    Run evaluation.
-
-    If `_set_global_evaluation` is `True`, sets the global evaluation to be run in another part of the program.
-
-    Otherwise, if there is no event loop, runs the evaluation in the current thread until completion.
+    If added to the file which is called through lmnr eval command, then simply registers the evaluation.
+    Otherwise, if there is no event loop, creates it and runs the evaluation until completion.
     If there is an event loop, schedules the evaluation as a task in the event loop and returns an awaitable handle.
+
+    Parameters:
+        data (Union[List[Union[EvaluationDatapoint, dict]], EvaluationDataset]): List of data points to evaluate or an evaluation dataset.
+                        `data` is the input to the executor function,
+                        `target` is the input to the evaluator function.
+        executor (Callable[..., Any]): The executor function.
+                        Takes the data point + any additional arguments
+                        and returns the output to evaluate.
+        evaluators (List[Callable[..., Any]]): List of evaluator functions.
+            Each evaluator function takes the output of the executor _and_
+            the target data, and returns a score. The score can be a
+            single number or a record of string keys and number values.
+            If the score is a single number, it will be named after the
+            evaluator function. If the function is anonymous, it will be
+            named `evaluator_${index}`, where index is the index of the
+            evaluator function in the list starting from 1.
+        name (Optional[str], optional): The name of the evaluation.
+            It will be auto-generated if not provided.
+        batch_size (int, optional): The batch size for evaluation.
+                        Defaults to DEFAULT_BATCH_SIZE.
+        project_api_key (Optional[str], optional): The project API key.
+                        Defaults to an empty string.
+        base_url (Optional[str], optional): The base URL for the Laminar API.
+                        Useful if self-hosted elsewhere.
+                        Defaults to "https://api.lmnr.ai".
+        http_port (Optional[int], optional): The port for the Laminar API HTTP service.
+                        Defaults to 443.
     """
 
     evaluation = Evaluation(
-        name,
-        data,
-        executor,
-        evaluators,
-        batch_size,
-        project_api_key,
-        base_url,
-        http_port,
+        data=data,
+        executor=executor,
+        evaluators=evaluators,
+        name=name,
+        batch_size=batch_size,
+        project_api_key=project_api_key,
+        base_url=base_url,
+        http_port=http_port,
     )
 
     global _evaluation

{lmnr-0.4.12b1 → lmnr-0.4.12b3}/src/lmnr/sdk/laminar.py

@@ -410,7 +410,7 @@ class Laminar:
         set_association_properties(props)
 
     @classmethod
-    def create_evaluation(cls, name: str) -> CreateEvaluationResponse:
+    def create_evaluation(cls, name: Optional[str]) -> CreateEvaluationResponse:
         response = requests.post(
             cls.__base_http_url + "/v1/evaluations",
             data=json.dumps({"name": name}),

{lmnr-0.4.12b1 → lmnr-0.4.12b3}/src/lmnr/sdk/types.py

@@ -18,6 +18,8 @@ class ConditionedValue(pydantic.BaseModel):
 
 
 Numeric = Union[int, float]
+NumericTypes = (int, float)  # for use with isinstance
+
 NodeInput = Union[str, list[ChatMessage], ConditionedValue, Numeric, bool]
 PipelineOutput = Union[NodeInput]