lmnr 0.4.11__tar.gz → 0.4.12b1__tar.gz

{lmnr-0.4.11 → lmnr-0.4.12b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.11
+Version: 0.4.12b1
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: argparse (>=1.0,<2.0)
 Requires-Dist: asyncio (>=3.0,<4.0)
 Requires-Dist: backoff (>=2.0,<3.0)
 Requires-Dist: colorama (>=0.4,<0.5)
@@ -54,6 +55,7 @@ Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: python-dotenv (>=1.0,<2.0)
 Requires-Dist: requests (>=2.0,<3.0)
 Requires-Dist: tenacity (>=8.0,<9.0)
+Requires-Dist: tqdm (>=4.0,<5.0)
 Description-Content-Type: text/markdown
 
 # Laminar Python

{lmnr-0.4.11 → lmnr-0.4.12b1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lmnr"
-version = "0.4.11"
+version = "0.4.12b1"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -11,7 +11,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.11"
+version = "0.4.12b1"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"
@@ -62,6 +62,8 @@ opentelemetry-instrumentation-weaviate = "^0.30.0"
 opentelemetry-instrumentation-alephalpha = "^0.30.0"
 opentelemetry-instrumentation-marqo = "^0.30.0"
 opentelemetry-instrumentation-groq = "^0.30.0"
+tqdm = "~=4.0"
+argparse = "~=1.0"
 
 [tool.poetry.group.dev.dependencies]
 autopep8 = "^2.2.0"
@@ -83,11 +85,8 @@ langchain-openai = "^0.1.15"
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
-[project.entry-points.console_scripts]
-lmnr = "lmnr.cli.cli:cli"
-
 [tool.poetry.scripts]
-lmnr = "lmnr.cli.cli:cli"
+lmnr = "lmnr.cli:cli"
 
 [project.optional-dependencies]
 test = ["pytest"]

{lmnr-0.4.11 → lmnr-0.4.12b1}/src/lmnr/__init__.py

@@ -1,4 +1,4 @@
-from .sdk.evaluations import Evaluation
+from .sdk.evaluations import evaluate
 from .sdk.laminar import Laminar
 from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput
 from .sdk.decorators import observe

lmnr-0.4.12b1/src/lmnr/cli.py

@@ -0,0 +1,39 @@
+from argparse import ArgumentParser
+import asyncio
+import importlib
+import os
+import sys
+
+from lmnr.sdk.evaluations import set_global_evaluation
+
+
+# TODO: Refactor this code
+async def run_evaluation(args):
+    sys.path.insert(0, os.getcwd())
+
+    with set_global_evaluation(True):
+        file = os.path.abspath(args.file)
+
+        spec = importlib.util.spec_from_file_location("run_eval", file)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+
+        from lmnr.sdk.evaluations import _evaluation
+        evaluation = _evaluation
+        await evaluation.run()
+
+
+def cli():
+    parser = ArgumentParser(
+        prog="lmnr",
+        description="CLI for Laminar",
+    )
+
+    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
+
+    parser_eval = subparsers.add_parser("eval", description="Run an evaluation")
+    parser_eval.add_argument("file", help="A file containing the evaluation to run")
+    parser_eval.set_defaults(func=run_evaluation)
+
+    parsed = parser.parse_args()
+    asyncio.run(parsed.func(parsed))

lmnr-0.4.12b1/src/lmnr/sdk/evaluations.py

@@ -0,0 +1,290 @@
+import asyncio
+import sys
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from typing import Any, Awaitable, Optional, Union
+
+from tqdm import tqdm
+
+from .laminar import Laminar as L
+from .types import CreateEvaluationResponse, Datapoint, EvaluationResultDatapoint, Numeric
+from .utils import is_async
+
+DEFAULT_BATCH_SIZE = 5
+
+_evaluation = None
+_set_global_evaluation = False
+
+
+@contextmanager
+def set_global_evaluation(set_global_evaluation: bool):
+    global _set_global_evaluation
+    original = _set_global_evaluation
+    try:
+        _set_global_evaluation = set_global_evaluation
+        yield
+    finally:
+        _set_global_evaluation = original
+        pass
+
+
+def get_evaluation_url(project_id: str, evaluation_id: str):
+    return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
+
+
+class EvaluationReporter:
+    def __init__(self):
+        pass
+
+    def start(self, name: str, project_id: str, id: str, length: int):
+        print(f"Running evaluation {name}...\n")
+        print(f"Check progress and results at {get_evaluation_url(project_id, id)}\n")
+        self.cli_progress = tqdm(total=length, bar_format="{bar} {percentage:3.0f}% | ETA: {remaining}s | {n_fmt}/{total_fmt}", ncols=60)
+
+    def update(self, batch_length: int):
+        self.cli_progress.update(batch_length)
+
+    def stopWithError(self, error: Exception):
+        self.cli_progress.close()
+        sys.stderr.write(f"\nError: {error}\n")
+
+    def stop(self, average_scores: dict[str, Numeric]):
+        self.cli_progress.close()
+        print("\nAverage scores:")
+        for (name, score) in average_scores.items():
+            print(f"{name}: {score}")
+        print("\n")
+
+
+class EvaluationDataset(ABC):
+    @abstractmethod
+    def __init__(self, *args, **kwargs):
+        pass
+
+    @abstractmethod
+    def __len__(self) -> int:
+        pass
+
+    @abstractmethod
+    def __getitem__(self, idx) -> Datapoint:
+        pass
+
+    def slice(self, start: int, end: int):
+        return [self[i] for i in range(max(start, 0), min(end, len(self)))]
+
+
+class Evaluation:
+    def __init__(
+        self,
+        name: str,
+        data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
+        executor: Any,
+        evaluators: list[Any],
+        batch_size: int = DEFAULT_BATCH_SIZE,
+        project_api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        http_port: Optional[int] = None,
+    ):
+        """
+        Initializes an instance of the Evaluations class.
+
+        Parameters:
+            name (str): The name of the evaluation.
+            data (Union[List[Union[EvaluationDatapoint, dict]], EvaluationDataset]): List of data points to evaluate or an evaluation dataset.
+                            `data` is the input to the executor function,
+                            `target` is the input to the evaluator function.
+            executor (Callable[..., Any]): The executor function.
+                            Takes the data point + any additional arguments
+                            and returns the output to evaluate.
+            evaluators (List[Callable[..., Any]]): List of evaluator functions.
+                Each evaluator function takes the output of the executor _and_
+                the target data, and returns a score. The score can be a
+                single number or a record of string keys and number values.
+                If the score is a single number, it will be named after the
+                evaluator function. If the function is anonymous, it will be
+                named `evaluator_${index}`, where index is the index of the
+                evaluator function in the list starting from 1.
+            batch_size (int, optional): The batch size for evaluation.
+                            Defaults to DEFAULT_BATCH_SIZE.
+            project_api_key (Optional[str], optional): The project API key.
+                            Defaults to an empty string.
+            base_url (Optional[str], optional): The base URL for the Laminar API.
+                            Useful if self-hosted elsewhere.
+                            Defaults to "https://api.lmnr.ai".
+            http_port (Optional[int], optional): The port for the Laminar API HTTP service.
+                            Defaults to 443.
+        """
+
+        self.is_finished = False
+        self.name = name
+        self.reporter = EvaluationReporter()
+        self.executor = executor
+        self.evaluators = dict(
+            zip(
+                [
+                    (
+                        e.__name__
+                        if e.__name__ and e.__name__ != "<lambda>"
+                        else f"evaluator_{i+1}"
+                    )
+                    for i, e in enumerate(evaluators)
+                ],
+                evaluators,
+            )
+        )
+        self.evaluator_names = list(self.evaluators.keys())
+        if isinstance(data, list):
+            self.data = [
+                (
+                    Datapoint.model_validate(point)
+                    if isinstance(point, dict)
+                    else point
+                )
+                for point in data
+            ]
+        else:
+            self.data = data
+        self.batch_size = batch_size
+        L.initialize(
+            project_api_key=project_api_key,
+            base_url=base_url,
+            http_port=http_port,
+            instruments=set(),
+        )
+
+    def run(self) -> Union[None, Awaitable[None]]:
+        """Runs the evaluation.
+
+        Creates a new evaluation if no evaluation with such name exists, or
+        adds data to an existing one otherwise. Evaluates data points in
+        batches of `self.batch_size`. The executor
+        function is called on each data point to get the output,
+        and then evaluate it by each evaluator function.
+
+        Usage:
+        ```python
+        # in a synchronous context:
+        e.run()
+        # in an asynchronous context:
+        await e.run()
+        ```
+
+        """
+        if self.is_finished:
+            raise Exception("Evaluation is already finished")
+
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            return loop.create_task(self._run())
+        else:
+            return loop.run_until_complete(self._run())
+
+    async def _run(self) -> None:
+        evaluation = L.create_evaluation(self.name)
+        self.reporter.start(
+            evaluation.name,
+            evaluation.projectId,
+            evaluation.id,
+            len(self.data),
+        )
+
+        try:
+            await self.evaluate_in_batches(evaluation)
+        except Exception as e:
+            L.update_evaluation_status(evaluation.id, "Error")
+            self.reporter.stopWithError(e)
+            self.is_finished = True
+            return
+
+        # If we update with status "Finished", we expect averageScores to be not empty
+        updated_evaluation = L.update_evaluation_status(evaluation.id, "Finished")
+        self.reporter.stop(updated_evaluation.averageScores)
+        self.is_finished = True
+
+    async def evaluate_in_batches(self, evaluation: CreateEvaluationResponse):
+        for i in range(0, len(self.data), self.batch_size):
+            batch = (
+                self.data[i: i + self.batch_size]
+                if isinstance(self.data, list)
+                else self.data.slice(i, i + self.batch_size)
+            )
+            try:
+                results = await self._evaluate_batch(batch)
+                L.post_evaluation_results(evaluation.id, results)
+            except Exception as e:
+                print(f"Error evaluating batch: {e}")
+            finally:
+                self.reporter.update(len(batch))
+
+    async def _evaluate_batch(self, batch: list[Datapoint]) -> list[EvaluationResultDatapoint]:
+        batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
+        results = await asyncio.gather(*batch_promises)
+        return results
+
+    async def _evaluate_datapoint(self, datapoint) -> EvaluationResultDatapoint:
+        output = (
+            await self.executor(datapoint.data)
+            if is_async(self.executor)
+            else self.executor(datapoint.data)
+        )
+        target = datapoint.target
+
+        # Iterate over evaluators
+        scores: dict[str, Numeric] = {}
+        for evaluator_name in self.evaluator_names:
+            evaluator = self.evaluators[evaluator_name]
+            value = (
+                await evaluator(output, target)
+                if is_async(evaluator)
+                else evaluator(output, target)
+            )
+
+            # If evaluator returns a single number, use evaluator name as key
+            if isinstance(value, Numeric):
+                scores[evaluator_name] = value
+            else:
+                scores.update(value)
+
+        return EvaluationResultDatapoint(
+            data=datapoint.data,
+            target=target,
+            executorOutput=output,
+            scores=scores,
+        )
+
+
+def evaluate(
+    name: str,
+    data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
+    executor: Any,
+    evaluators: list[Any],
+    batch_size: int = DEFAULT_BATCH_SIZE,
+    project_api_key: Optional[str] = None,
+    base_url: Optional[str] = None,
+    http_port: Optional[int] = None,
+) -> Optional[Awaitable[None]]:
+    """
+    Run evaluation.
+
+    If `_set_global_evaluation` is `True`, sets the global evaluation to be run in another part of the program.
+
+    Otherwise, if there is no event loop, runs the evaluation in the current thread until completion.
+    If there is an event loop, schedules the evaluation as a task in the event loop and returns an awaitable handle.
+    """
+
+    evaluation = Evaluation(
+        name,
+        data,
+        executor,
+        evaluators,
+        batch_size,
+        project_api_key,
+        base_url,
+        http_port,
+    )
+
+    global _evaluation
+    if _set_global_evaluation:
+        _evaluation = evaluation
+    else:
+        return evaluation.run()

{lmnr-0.4.11 → lmnr-0.4.12b1}/src/lmnr/sdk/laminar.py

@@ -37,11 +37,13 @@ from .types import (
     PipelineRunResponse,
     NodeInput,
     PipelineRunRequest,
+    UpdateEvaluationResponse,
 )
 
 
 class Laminar:
-    __base_url: str = "https://api.lmnr.ai:8443"
+    __base_http_url: str
+    __base_grpc_url: str
     __project_api_key: Optional[str] = None
     __env: dict[str, str] = {}
     __initialized: bool = False
@@ -52,6 +54,8 @@ class Laminar:
         project_api_key: Optional[str] = None,
         env: dict[str, str] = {},
         base_url: Optional[str] = None,
+        http_port: Optional[int] = None,
+        grpc_port: Optional[int] = None,
         instruments: Optional[Set[Instruments]] = None,
     ):
         """Initialize Laminar context across the application.
@@ -71,13 +75,12 @@ class Laminar:
                             overriden at request time. Usually, model
                             provider keys are stored here.
                             Defaults to {}.
-            base_url (Optional[str], optional): Url of Laminar endpoint,
-                            or the  customopen telemetry ingester.
-                            If not specified, defaults to
-                            https://api.lmnr.ai:8443.
-                            For locally hosted Laminar, default setting
-                            must be http://localhost:8001
-                            Defaults to None.
+            base_url (Optional[str], optional): Laminar API url.
+                            If not specified, defaults to https://api.lmnr.ai.
+            http_port (Optional[int], optional): Laminar API http port.
+                            If not specified, defaults to 443.
+            grpc_port (Optional[int], optional): Laminar API grpc port.
+                            If not specified, defaults to 8443.
 
         Raises:
             ValueError: If project API key is not set
@@ -85,7 +88,7 @@ class Laminar:
         cls.__project_api_key = project_api_key or os.environ.get(
             "LMNR_PROJECT_API_KEY"
         )
-        if not project_api_key:
+        if not cls.__project_api_key:
             dotenv_path = dotenv.find_dotenv(usecwd=True)
             cls.__project_api_key = dotenv.get_key(
                 dotenv_path=dotenv_path, key_to_get="LMNR_PROJECT_API_KEY"
@@ -96,14 +99,16 @@ class Laminar:
                 " your project API key or set the LMNR_PROJECT_API_KEY"
                 " environment variable in your environment or .env file"
             )
-        if base_url is not None:
-            cls.__base_url = base_url
+
+        cls.__base_http_url = f"{base_url or 'https://api.lmnr.ai'}:{http_port or 443}"
+        cls.__base_grpc_url = f"{base_url or 'https://api.lmnr.ai'}:{grpc_port or 8443}"
+
         cls.__env = env
         cls.__initialized = True
         cls._initialize_logger()
         Traceloop.init(
             exporter=OTLPSpanExporter(
-                endpoint=cls.__base_url,
+                endpoint=cls.__base_grpc_url,
                 headers={"authorization": f"Bearer {cls.__project_api_key}"},
             ),
             instruments=instruments,
@@ -190,7 +195,7 @@ class Laminar:
             raise ValueError(f"Invalid request: {e}")
 
         response = requests.post(
-            cls.__base_url + "/v1/pipeline/run",
+            cls.__base_http_url + "/v1/pipeline/run",
             data=json.dumps(request.to_dict()),
             headers=cls._headers(),
         )
@@ -292,7 +297,7 @@ class Laminar:
             )
             return
 
-        current_span.add_event(name, event)
+        current_span.add_event(name, event, timestamp)
 
     @classmethod
     @contextmanager
@@ -407,7 +412,7 @@ class Laminar:
     @classmethod
     def create_evaluation(cls, name: str) -> CreateEvaluationResponse:
         response = requests.post(
-            cls.__base_url + "/v1/evaluations",
+            cls.__base_http_url + "/v1/evaluations",
             data=json.dumps({"name": name}),
             headers=cls._headers(),
         )
@@ -421,14 +426,14 @@ class Laminar:
 
     @classmethod
     def post_evaluation_results(
-        cls, evaluation_name: str, data: list[EvaluationResultDatapoint]
+        cls, evaluation_id: uuid.UUID, data: list[EvaluationResultDatapoint]
     ) -> requests.Response:
         body = {
-            "name": evaluation_name,
-            "points": data,
+            "evaluationId": str(evaluation_id),
+            "points": [datapoint.model_dump() for datapoint in data],
         }
         response = requests.post(
-            cls.__base_url + "/v1/evaluation-datapoints",
+            cls.__base_http_url + "/v1/evaluation-datapoints",
             data=json.dumps(body),
             headers=cls._headers(),
         )
@@ -446,28 +451,38 @@ class Laminar:
 
     @classmethod
     def update_evaluation_status(
-        cls, evaluation_name: str, status: str
-    ) -> requests.Response:
+        cls, evaluation_id: str, status: str
+    ) -> UpdateEvaluationResponse:
+        """
+        Updates the status of an evaluation. Returns the updated evaluation object.
+
+        Args:
+            evaluation_id (str): The ID of the evaluation to update.
+            status (str): The status to set for the evaluation.
+
+        Returns:
+            UpdateEvaluationResponse: The updated evaluation response.
+
+        Raises:
+            ValueError: If the request fails.
+        """
         body = {
-            "name": evaluation_name,
             "status": status,
         }
-        response = requests.put(
-            cls.__base_url + "/v1/evaluations/",
+        url = f"{cls.__base_http_url}/v1/evaluations/{evaluation_id}"
+
+        response = requests.post(
+            url,
             data=json.dumps(body),
             headers=cls._headers(),
         )
         if response.status_code != 200:
-            try:
-                resp_json = response.json()
-                raise ValueError(
-                    f"Failed to send evaluation status. Response: {json.dumps(resp_json)}"
-                )
-            except Exception:
-                raise ValueError(
-                    f"Failed to send evaluation status. Error: {response.text}"
-                )
-        return response
+            raise ValueError(
+                f"Failed to update evaluation status {evaluation_id}. "
+                f"Response: {response.text}"
+            )
+
+        return UpdateEvaluationResponse.model_validate(response.json())
 
     @classmethod
     def _headers(cls):

{lmnr-0.4.11 → lmnr-0.4.12b1}/src/lmnr/sdk/types.py

@@ -79,7 +79,7 @@ EvaluationDatapointTarget = dict[str, Any]
 
 
 # EvaluationDatapoint is a single data point in the evaluation
-class EvaluationDatapoint(pydantic.BaseModel):
+class Datapoint(pydantic.BaseModel):
     # input to the executor function. Must be a dict with string keys
     data: EvaluationDatapointData
     # input to the evaluator function (alongside the executor output).
@@ -114,10 +114,14 @@ class CreateEvaluationResponse(pydantic.BaseModel):
     status: EvaluationStatus
     projectId: uuid.UUID
     metadata: Optional[dict[str, Any]] = None
+    averageScores: Optional[dict[str, Numeric]] = None
+
+
+UpdateEvaluationResponse = CreateEvaluationResponse
 
 
 class EvaluationResultDatapoint(pydantic.BaseModel):
     data: EvaluationDatapointData
     target: EvaluationDatapointTarget
-    executor_output: ExecutorFunctionReturnType
+    executorOutput: ExecutorFunctionReturnType
     scores: dict[str, Numeric]

{lmnr-0.4.11 → lmnr-0.4.12b1}/src/lmnr/traceloop_sdk/__init__.py

@@ -55,9 +55,6 @@ class Traceloop:
 
         enable_content_tracing = is_content_tracing_enabled()
 
-        if exporter or processor:
-            print(Fore.GREEN + "Laminar exporting traces to a custom exporter")
-
         headers = os.getenv("TRACELOOP_HEADERS") or headers
 
         if isinstance(headers, str):
@@ -78,17 +75,7 @@ class Traceloop:
             print(Fore.RESET)
             return
 
-        if not exporter and not processor and headers:
-            print(
-                Fore.GREEN
-                + f"Laminar exporting traces to {api_endpoint}, authenticating with custom headers"
-            )
-
         if api_key and not exporter and not processor and not headers:
-            print(
-                Fore.GREEN
-                + f"Laminar exporting traces to {api_endpoint} authenticating with bearer token"
-            )
             headers = {
                 "Authorization": f"Bearer {api_key}",
             }

{lmnr-0.4.11 → lmnr-0.4.12b1}/src/lmnr/traceloop_sdk/tracing/tracing.py

@@ -124,46 +124,34 @@ class TracerWrapper(object):
             # this makes sure otel context is propagated so we always want it
             ThreadingInstrumentor().instrument()
 
-            instrument_set = False
             if instruments is None:
                 init_instrumentations(should_enrich_metrics)
-                instrument_set = True
             else:
                 for instrument in instruments:
                     if instrument == Instruments.OPENAI:
                         if not init_openai_instrumentor(should_enrich_metrics):
                             print(Fore.RED + "Warning: OpenAI library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.ANTHROPIC:
                         if not init_anthropic_instrumentor(should_enrich_metrics):
                             print(
                                 Fore.RED + "Warning: Anthropic library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.COHERE:
                         if not init_cohere_instrumentor():
                             print(Fore.RED + "Warning: Cohere library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.PINECONE:
                         if not init_pinecone_instrumentor():
                             print(
                                 Fore.RED + "Warning: Pinecone library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.CHROMA:
                         if not init_chroma_instrumentor():
                             print(Fore.RED + "Warning: Chroma library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.GOOGLE_GENERATIVEAI:
                         if not init_google_generativeai_instrumentor():
                             print(
@@ -171,44 +159,32 @@ class TracerWrapper(object):
                                 + "Warning: Google Generative AI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.LANGCHAIN:
                         if not init_langchain_instrumentor():
                             print(
                                 Fore.RED + "Warning: LangChain library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.MISTRAL:
                         if not init_mistralai_instrumentor():
                             print(
                                 Fore.RED + "Warning: MistralAI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.OLLAMA:
                         if not init_ollama_instrumentor():
                             print(Fore.RED + "Warning: Ollama library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.LLAMA_INDEX:
                         if not init_llama_index_instrumentor():
                             print(
                                 Fore.RED + "Warning: LlamaIndex library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.MILVUS:
                         if not init_milvus_instrumentor():
                             print(Fore.RED + "Warning: Milvus library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.TRANSFORMERS:
                         if not init_transformers_instrumentor():
                             print(
@@ -216,72 +192,52 @@ class TracerWrapper(object):
                                 + "Warning: Transformers library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.TOGETHER:
                         if not init_together_instrumentor():
                             print(
                                 Fore.RED + "Warning: TogetherAI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.REQUESTS:
                         if not init_requests_instrumentor():
                             print(
                                 Fore.RED + "Warning: Requests library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.URLLIB3:
                         if not init_urllib3_instrumentor():
                             print(Fore.RED + "Warning: urllib3 library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.PYMYSQL:
                         if not init_pymysql_instrumentor():
                             print(Fore.RED + "Warning: PyMySQL library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.BEDROCK:
                         if not init_bedrock_instrumentor(should_enrich_metrics):
                             print(Fore.RED + "Warning: Bedrock library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.REPLICATE:
                         if not init_replicate_instrumentor():
                             print(
                                 Fore.RED + "Warning: Replicate library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.VERTEXAI:
                         if not init_vertexai_instrumentor():
                             print(
                                 Fore.RED + "Warning: Vertex AI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.WATSONX:
                         if not init_watsonx_instrumentor():
                             print(Fore.RED + "Warning: Watsonx library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.WEAVIATE:
                         if not init_weaviate_instrumentor():
                             print(
                                 Fore.RED + "Warning: Weaviate library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.ALEPHALPHA:
                         if not init_alephalpha_instrumentor():
                             print(
@@ -289,26 +245,18 @@ class TracerWrapper(object):
                                 + "Warning: Aleph Alpha library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.MARQO:
                         if not init_marqo_instrumentor():
                             print(Fore.RED + "Warning: marqo library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.LANCEDB:
                         if not init_lancedb_instrumentor():
                             print(Fore.RED + "Warning: LanceDB library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.REDIS:
                         if not init_redis_instrumentor():
                             print(Fore.RED + "Warning: redis library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
 
                     else:
                         print(
@@ -324,13 +272,6 @@ class TracerWrapper(object):
                         )
                         print(Fore.RESET)
 
-            if not instrument_set:
-                print(
-                    Fore.RED + "Warning: No valid instruments set. Remove 'instrument' "
-                    "argument to use all instruments, or set a valid instrument."
-                )
-                print(Fore.RESET)
-
             obj.__content_allow_list = ContentAllowList()
 
             # Force flushes for debug environments (e.g. local development)

lmnr-0.4.11/src/lmnr/sdk/evaluations.py

@@ -1,178 +0,0 @@
-from typing import Any, Union
-
-from .types import EvaluationDatapoint
-from .utils import is_async
-from .laminar import Laminar as L
-import asyncio
-
-from abc import ABC, abstractmethod
-
-DEFAULT_BATCH_SIZE = 5
-
-
-class EvaluationDataset(ABC):
-    @abstractmethod
-    def __init__(self, *args, **kwargs):
-        pass
-
-    @abstractmethod
-    def __len__(self) -> int:
-        pass
-
-    @abstractmethod
-    def __getitem__(self, idx) -> EvaluationDatapoint:
-        pass
-
-    def slice(self, start: int, end: int):
-        return [self[i] for i in range(max(start, 0), min(end, len(self)))]
-
-
-class Evaluation:
-    def __init__(
-        self,
-        name,
-        data: Union[EvaluationDataset, list[Union[EvaluationDatapoint, dict]]],
-        executor: Any,
-        evaluators: list[Any],
-        batch_size: int = DEFAULT_BATCH_SIZE,
-        project_api_key: str = "",
-        base_url: str = "https://api.lmnr.ai",
-    ):
-        """
-        Initializes an instance of the Evaluations class.
-        Parameters:
-            name (str): The name of the evaluation.
-            data (Union[List[Union[EvaluationDatapoint, dict]], EvaluationDataset]): List of data points to evaluate or an evaluation dataset.
-                            `data` is the input to the executor function,
-                            `target` is the input to the evaluator function.
-            executor (Callable[..., Any]): The executor function.
-                            Takes the data point + any additional arguments
-                            and returns the output to evaluate.
-            evaluators (List[Callable[..., Any]]): List of evaluator functions.
-                Each evaluator function takes the output of the executor _and_
-                the target data, and returns a score. The score can be a
-                single number or a record of string keys and number values.
-                If the score is a single number, it will be named after the
-                evaluator function. If the function is anonymous, it will be
-                named `evaluator_${index}`, where index is the index of the
-                evaluator function in the list starting from 1.
-            batch_size (int, optional): The batch size for evaluation.
-                            Defaults to DEFAULT_BATCH_SIZE.
-            project_api_key (str, optional): The project API key.
-                            Defaults to an empty string.
-            base_url (str, optional): The base URL for the LMNR API.
-                            Useful if self-hosted elsewhere.
-                            Defaults to "https://api.lmnr.ai".
-        """
-
-        self.name = name
-        self.executor = executor
-        self.evaluators = dict(
-            zip(
-                [
-                    (
-                        e.__name__
-                        if e.__name__ and e.__name__ != "<lambda>"
-                        else f"evaluator_{i+1}"
-                    )
-                    for i, e in enumerate(evaluators)
-                ],
-                evaluators,
-            )
-        )
-        self.evaluator_names = list(self.evaluators.keys())
-        if isinstance(data, list):
-            self.data = [
-                (
-                    EvaluationDatapoint.model_validate(point)
-                    if isinstance(point, dict)
-                    else point
-                )
-                for point in data
-            ]
-        else:
-            self.data = data
-        self.batch_size = batch_size
-        L.initialize(project_api_key=project_api_key, base_url=base_url)
-
-    def run(self):
-        """Runs the evaluation.
-
-        Creates a new evaluation if no evaluation with such name exists, or
-        adds data to an existing one otherwise. Evaluates data points in
-        batches of `self.batch_size`. The executor
-        function is called on each data point to get the output,
-        and then evaluate it by each evaluator function.
-
-        Usage:
-        ```python
-        # in a synchronous context:
-        e.run()
-        # in an asynchronous context:
-        await e.run()
-        ```
-
-        """
-        loop = asyncio.get_event_loop()
-        if loop.is_running():
-            return loop.create_task(self._run())
-        else:
-            return loop.run_until_complete(self._run())
-
-    async def _run(self):
-        response = L.create_evaluation(self.name)
-
-        # Process batches sequentially
-        for i in range(0, len(self.data), self.batch_size):
-            batch = (
-                self.data[i : i + self.batch_size]
-                if isinstance(self.data, list)
-                else self.data.slice(i, i + self.batch_size)
-            )
-            try:
-                await self._evaluate_batch(batch)
-            except Exception as e:
-                print(f"Error evaluating batch: {e}")
-
-        try:
-            L.update_evaluation_status(response.name, "Finished")
-            print(f"Evaluation {response.id} complete")
-        except Exception as e:
-            print(f"Error updating evaluation status: {e}")
-
-    async def _evaluate_batch(self, batch: list[EvaluationDatapoint]):
-        batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
-        results = await asyncio.gather(*batch_promises)
-
-        return L.post_evaluation_results(self.name, results)
-
-    async def _evaluate_datapoint(self, datapoint):
-        output = (
-            await self.executor(datapoint.data)
-            if is_async(self.executor)
-            else self.executor(datapoint.data)
-        )
-        target = datapoint.target
-
-        # Iterate over evaluators
-        scores = {}
-        for evaluator_name in self.evaluator_names:
-            evaluator = self.evaluators[evaluator_name]
-            value = (
-                await evaluator(output, target)
-                if is_async(evaluator)
-                else evaluator(output, target)
-            )
-
-            # If evaluator returns a single number, use evaluator name as key
-            if isinstance(value, (int, float)):
-                scores[evaluator_name] = value
-            else:
-                scores.update(value)
-
-        return {
-            "executorOutput": output,
-            "data": datapoint.data,
-            "target": target,
-            "scores": scores,
-        }