lmnr 0.4.10__py3-none-any.whl → 0.4.12b1__py3-none-any.whl

lmnr/__init__.py

@@ -1,4 +1,4 @@
-from .sdk.evaluations import Evaluation
+from .sdk.evaluations import evaluate
 from .sdk.laminar import Laminar
 from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput
 from .sdk.decorators import observe

lmnr/cli.py

@@ -0,0 +1,39 @@
+from argparse import ArgumentParser
+import asyncio
+import importlib
+import os
+import sys
+
+from lmnr.sdk.evaluations import set_global_evaluation
+
+
+# TODO: Refactor this code
+async def run_evaluation(args):
+    sys.path.insert(0, os.getcwd())
+
+    with set_global_evaluation(True):
+        file = os.path.abspath(args.file)
+
+        spec = importlib.util.spec_from_file_location("run_eval", file)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+
+        from lmnr.sdk.evaluations import _evaluation
+        evaluation = _evaluation
+        await evaluation.run()
+
+
+def cli():
+    parser = ArgumentParser(
+        prog="lmnr",
+        description="CLI for Laminar",
+    )
+
+    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
+
+    parser_eval = subparsers.add_parser("eval", description="Run an evaluation")
+    parser_eval.add_argument("file", help="A file containing the evaluation to run")
+    parser_eval.set_defaults(func=run_evaluation)
+
+    parsed = parser.parse_args()
+    asyncio.run(parsed.func(parsed))

lmnr/sdk/decorators.py

@@ -4,22 +4,19 @@ from lmnr.traceloop_sdk.decorators.base import (
 )
 from opentelemetry.trace import INVALID_SPAN, get_current_span
 
-from typing import Callable, Optional, ParamSpec, TypeVar, cast
+from typing import Callable, Optional, cast
 
 from lmnr.traceloop_sdk.tracing.tracing import update_association_properties
 
 from .utils import is_async
 
-P = ParamSpec("P")
-R = TypeVar("R")
-
 
 def observe(
     *,
     name: Optional[str] = None,
     user_id: Optional[str] = None,
     session_id: Optional[str] = None,
-) -> Callable[[Callable[P, R]], Callable[P, R]]:
+) -> Callable[[Callable], Callable]:
     """The main decorator entrypoint for Laminar. This is used to wrap
     functions and methods to create spans.
 
@@ -41,7 +38,7 @@ def observe(
         R: Returns the result of the wrapped function
     """
 
-    def decorator(func: Callable[P, R]) -> Callable[P, R]:
+    def decorator(func: Callable) -> Callable:
         current_span = get_current_span()
         if current_span != INVALID_SPAN:
             if session_id is not None:
@@ -64,4 +61,4 @@ def observe(
             else entity_method(name=name)(func)
         )
 
-    return cast(Callable[P, R], decorator)
+    return cast(Callable, decorator)

lmnr/sdk/evaluations.py

@@ -1,14 +1,60 @@
-from typing import Any, Union
-
-from .types import EvaluationDatapoint
-from .utils import is_async
-from .laminar import Laminar as L
 import asyncio
-
+import sys
 from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from typing import Any, Awaitable, Optional, Union
+
+from tqdm import tqdm
+
+from .laminar import Laminar as L
+from .types import CreateEvaluationResponse, Datapoint, EvaluationResultDatapoint, Numeric
+from .utils import is_async
 
 DEFAULT_BATCH_SIZE = 5
 
+_evaluation = None
+_set_global_evaluation = False
+
+
+@contextmanager
+def set_global_evaluation(set_global_evaluation: bool):
+    global _set_global_evaluation
+    original = _set_global_evaluation
+    try:
+        _set_global_evaluation = set_global_evaluation
+        yield
+    finally:
+        _set_global_evaluation = original
+        pass
+
+
+def get_evaluation_url(project_id: str, evaluation_id: str):
+    return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
+
+
+class EvaluationReporter:
+    def __init__(self):
+        pass
+
+    def start(self, name: str, project_id: str, id: str, length: int):
+        print(f"Running evaluation {name}...\n")
+        print(f"Check progress and results at {get_evaluation_url(project_id, id)}\n")
+        self.cli_progress = tqdm(total=length, bar_format="{bar} {percentage:3.0f}% | ETA: {remaining}s | {n_fmt}/{total_fmt}", ncols=60)
+
+    def update(self, batch_length: int):
+        self.cli_progress.update(batch_length)
+
+    def stopWithError(self, error: Exception):
+        self.cli_progress.close()
+        sys.stderr.write(f"\nError: {error}\n")
+
+    def stop(self, average_scores: dict[str, Numeric]):
+        self.cli_progress.close()
+        print("\nAverage scores:")
+        for (name, score) in average_scores.items():
+            print(f"{name}: {score}")
+        print("\n")
+
 
 class EvaluationDataset(ABC):
     @abstractmethod
@@ -20,7 +66,7 @@ class EvaluationDataset(ABC):
         pass
 
     @abstractmethod
-    def __getitem__(self, idx) -> EvaluationDatapoint:
+    def __getitem__(self, idx) -> Datapoint:
         pass
 
     def slice(self, start: int, end: int):
@@ -30,16 +76,18 @@ class EvaluationDataset(ABC):
 class Evaluation:
     def __init__(
         self,
-        name,
-        data: Union[EvaluationDataset, list[Union[EvaluationDatapoint, dict]]],
+        name: str,
+        data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
         executor: Any,
         evaluators: list[Any],
         batch_size: int = DEFAULT_BATCH_SIZE,
-        project_api_key: str = "",
-        base_url: str = "https://api.lmnr.ai",
+        project_api_key: Optional[str] = None,
+        base_url: Optional[str] = None,
+        http_port: Optional[int] = None,
     ):
         """
         Initializes an instance of the Evaluations class.
+
         Parameters:
             name (str): The name of the evaluation.
             data (Union[List[Union[EvaluationDatapoint, dict]], EvaluationDataset]): List of data points to evaluate or an evaluation dataset.
@@ -58,14 +106,18 @@ class Evaluation:
                 evaluator function in the list starting from 1.
             batch_size (int, optional): The batch size for evaluation.
                             Defaults to DEFAULT_BATCH_SIZE.
-            project_api_key (str, optional): The project API key.
+            project_api_key (Optional[str], optional): The project API key.
                             Defaults to an empty string.
-            base_url (str, optional): The base URL for the LMNR API.
+            base_url (Optional[str], optional): The base URL for the Laminar API.
                             Useful if self-hosted elsewhere.
                             Defaults to "https://api.lmnr.ai".
+            http_port (Optional[int], optional): The port for the Laminar API HTTP service.
+                            Defaults to 443.
         """
 
+        self.is_finished = False
         self.name = name
+        self.reporter = EvaluationReporter()
         self.executor = executor
         self.evaluators = dict(
             zip(
@@ -84,7 +136,7 @@ class Evaluation:
         if isinstance(data, list):
             self.data = [
                 (
-                    EvaluationDatapoint.model_validate(point)
+                    Datapoint.model_validate(point)
                     if isinstance(point, dict)
                     else point
                 )
@@ -93,9 +145,14 @@ class Evaluation:
         else:
             self.data = data
         self.batch_size = batch_size
-        L.initialize(project_api_key=project_api_key, base_url=base_url)
+        L.initialize(
+            project_api_key=project_api_key,
+            base_url=base_url,
+            http_port=http_port,
+            instruments=set(),
+        )
 
-    def run(self):
+    def run(self) -> Union[None, Awaitable[None]]:
         """Runs the evaluation.
 
         Creates a new evaluation if no evaluation with such name exists, or
@@ -113,40 +170,58 @@ class Evaluation:
         ```
 
         """
+        if self.is_finished:
+            raise Exception("Evaluation is already finished")
+
         loop = asyncio.get_event_loop()
         if loop.is_running():
             return loop.create_task(self._run())
         else:
             return loop.run_until_complete(self._run())
 
-    async def _run(self):
-        response = L.create_evaluation(self.name)
+    async def _run(self) -> None:
+        evaluation = L.create_evaluation(self.name)
+        self.reporter.start(
+            evaluation.name,
+            evaluation.projectId,
+            evaluation.id,
+            len(self.data),
+        )
 
-        # Process batches sequentially
+        try:
+            await self.evaluate_in_batches(evaluation)
+        except Exception as e:
+            L.update_evaluation_status(evaluation.id, "Error")
+            self.reporter.stopWithError(e)
+            self.is_finished = True
+            return
+
+        # If we update with status "Finished", we expect averageScores to be not empty
+        updated_evaluation = L.update_evaluation_status(evaluation.id, "Finished")
+        self.reporter.stop(updated_evaluation.averageScores)
+        self.is_finished = True
+
+    async def evaluate_in_batches(self, evaluation: CreateEvaluationResponse):
         for i in range(0, len(self.data), self.batch_size):
             batch = (
-                self.data[i : i + self.batch_size]
+                self.data[i: i + self.batch_size]
                 if isinstance(self.data, list)
                 else self.data.slice(i, i + self.batch_size)
             )
             try:
-                await self._evaluate_batch(batch)
+                results = await self._evaluate_batch(batch)
+                L.post_evaluation_results(evaluation.id, results)
             except Exception as e:
                 print(f"Error evaluating batch: {e}")
+            finally:
+                self.reporter.update(len(batch))
 
-        try:
-            L.update_evaluation_status(response.name, "Finished")
-            print(f"Evaluation {response.id} complete")
-        except Exception as e:
-            print(f"Error updating evaluation status: {e}")
-
-    async def _evaluate_batch(self, batch: list[EvaluationDatapoint]):
+    async def _evaluate_batch(self, batch: list[Datapoint]) -> list[EvaluationResultDatapoint]:
         batch_promises = [self._evaluate_datapoint(datapoint) for datapoint in batch]
         results = await asyncio.gather(*batch_promises)
+        return results
 
-        return L.post_evaluation_results(self.name, results)
-
-    async def _evaluate_datapoint(self, datapoint):
+    async def _evaluate_datapoint(self, datapoint) -> EvaluationResultDatapoint:
         output = (
             await self.executor(datapoint.data)
             if is_async(self.executor)
@@ -155,7 +230,7 @@ class Evaluation:
         target = datapoint.target
 
         # Iterate over evaluators
-        scores = {}
+        scores: dict[str, Numeric] = {}
         for evaluator_name in self.evaluator_names:
             evaluator = self.evaluators[evaluator_name]
             value = (
@@ -165,14 +240,51 @@ class Evaluation:
             )
 
             # If evaluator returns a single number, use evaluator name as key
-            if isinstance(value, (int, float)):
+            if isinstance(value, Numeric):
                 scores[evaluator_name] = value
             else:
                 scores.update(value)
 
-        return {
-            "executorOutput": output,
-            "data": datapoint.data,
-            "target": target,
-            "scores": scores,
-        }
+        return EvaluationResultDatapoint(
+            data=datapoint.data,
+            target=target,
+            executorOutput=output,
+            scores=scores,
+        )
+
+
+def evaluate(
+    name: str,
+    data: Union[EvaluationDataset, list[Union[Datapoint, dict]]],
+    executor: Any,
+    evaluators: list[Any],
+    batch_size: int = DEFAULT_BATCH_SIZE,
+    project_api_key: Optional[str] = None,
+    base_url: Optional[str] = None,
+    http_port: Optional[int] = None,
+) -> Optional[Awaitable[None]]:
+    """
+    Run evaluation.
+
+    If `_set_global_evaluation` is `True`, sets the global evaluation to be run in another part of the program.
+
+    Otherwise, if there is no event loop, runs the evaluation in the current thread until completion.
+    If there is an event loop, schedules the evaluation as a task in the event loop and returns an awaitable handle.
+    """
+
+    evaluation = Evaluation(
+        name,
+        data,
+        executor,
+        evaluators,
+        batch_size,
+        project_api_key,
+        base_url,
+        http_port,
+    )
+
+    global _evaluation
+    if _set_global_evaluation:
+        _evaluation = evaluation
+    else:
+        return evaluation.run()

lmnr/sdk/laminar.py

@@ -37,11 +37,13 @@ from .types import (
     PipelineRunResponse,
     NodeInput,
     PipelineRunRequest,
+    UpdateEvaluationResponse,
 )
 
 
 class Laminar:
-    __base_url: str = "https://api.lmnr.ai:8443"
+    __base_http_url: str
+    __base_grpc_url: str
     __project_api_key: Optional[str] = None
     __env: dict[str, str] = {}
     __initialized: bool = False
@@ -52,6 +54,8 @@ class Laminar:
         project_api_key: Optional[str] = None,
         env: dict[str, str] = {},
         base_url: Optional[str] = None,
+        http_port: Optional[int] = None,
+        grpc_port: Optional[int] = None,
         instruments: Optional[Set[Instruments]] = None,
     ):
         """Initialize Laminar context across the application.
@@ -71,13 +75,12 @@ class Laminar:
                             overriden at request time. Usually, model
                             provider keys are stored here.
                             Defaults to {}.
-            base_url (Optional[str], optional): Url of Laminar endpoint,
-                            or the  customopen telemetry ingester.
-                            If not specified, defaults to
-                            https://api.lmnr.ai:8443.
-                            For locally hosted Laminar, default setting
-                            must be http://localhost:8001
-                            Defaults to None.
+            base_url (Optional[str], optional): Laminar API url.
+                            If not specified, defaults to https://api.lmnr.ai.
+            http_port (Optional[int], optional): Laminar API http port.
+                            If not specified, defaults to 443.
+            grpc_port (Optional[int], optional): Laminar API grpc port.
+                            If not specified, defaults to 8443.
 
         Raises:
             ValueError: If project API key is not set
@@ -85,7 +88,7 @@ class Laminar:
         cls.__project_api_key = project_api_key or os.environ.get(
             "LMNR_PROJECT_API_KEY"
         )
-        if not project_api_key:
+        if not cls.__project_api_key:
             dotenv_path = dotenv.find_dotenv(usecwd=True)
             cls.__project_api_key = dotenv.get_key(
                 dotenv_path=dotenv_path, key_to_get="LMNR_PROJECT_API_KEY"
@@ -96,14 +99,16 @@ class Laminar:
                 " your project API key or set the LMNR_PROJECT_API_KEY"
                 " environment variable in your environment or .env file"
             )
-        if base_url is not None:
-            cls.__base_url = base_url
+
+        cls.__base_http_url = f"{base_url or 'https://api.lmnr.ai'}:{http_port or 443}"
+        cls.__base_grpc_url = f"{base_url or 'https://api.lmnr.ai'}:{grpc_port or 8443}"
+
         cls.__env = env
         cls.__initialized = True
         cls._initialize_logger()
         Traceloop.init(
             exporter=OTLPSpanExporter(
-                endpoint=cls.__base_url,
+                endpoint=cls.__base_grpc_url,
                 headers={"authorization": f"Bearer {cls.__project_api_key}"},
             ),
             instruments=instruments,
@@ -190,7 +195,7 @@ class Laminar:
             raise ValueError(f"Invalid request: {e}")
 
         response = requests.post(
-            cls.__base_url + "/v1/pipeline/run",
+            cls.__base_http_url + "/v1/pipeline/run",
             data=json.dumps(request.to_dict()),
             headers=cls._headers(),
         )
@@ -292,7 +297,7 @@ class Laminar:
             )
             return
 
-        current_span.add_event(name, event)
+        current_span.add_event(name, event, timestamp)
 
     @classmethod
     @contextmanager
@@ -407,7 +412,7 @@ class Laminar:
     @classmethod
     def create_evaluation(cls, name: str) -> CreateEvaluationResponse:
         response = requests.post(
-            cls.__base_url + "/v1/evaluations",
+            cls.__base_http_url + "/v1/evaluations",
             data=json.dumps({"name": name}),
             headers=cls._headers(),
         )
@@ -421,14 +426,14 @@ class Laminar:
 
     @classmethod
     def post_evaluation_results(
-        cls, evaluation_name: str, data: list[EvaluationResultDatapoint]
+        cls, evaluation_id: uuid.UUID, data: list[EvaluationResultDatapoint]
     ) -> requests.Response:
         body = {
-            "name": evaluation_name,
-            "points": data,
+            "evaluationId": str(evaluation_id),
+            "points": [datapoint.model_dump() for datapoint in data],
         }
         response = requests.post(
-            cls.__base_url + "/v1/evaluation-datapoints",
+            cls.__base_http_url + "/v1/evaluation-datapoints",
             data=json.dumps(body),
             headers=cls._headers(),
         )
@@ -446,28 +451,38 @@ class Laminar:
 
     @classmethod
     def update_evaluation_status(
-        cls, evaluation_name: str, status: str
-    ) -> requests.Response:
+        cls, evaluation_id: str, status: str
+    ) -> UpdateEvaluationResponse:
+        """
+        Updates the status of an evaluation. Returns the updated evaluation object.
+
+        Args:
+            evaluation_id (str): The ID of the evaluation to update.
+            status (str): The status to set for the evaluation.
+
+        Returns:
+            UpdateEvaluationResponse: The updated evaluation response.
+
+        Raises:
+            ValueError: If the request fails.
+        """
         body = {
-            "name": evaluation_name,
             "status": status,
         }
-        response = requests.put(
-            cls.__base_url + "/v1/evaluations/",
+        url = f"{cls.__base_http_url}/v1/evaluations/{evaluation_id}"
+
+        response = requests.post(
+            url,
             data=json.dumps(body),
             headers=cls._headers(),
         )
         if response.status_code != 200:
-            try:
-                resp_json = response.json()
-                raise ValueError(
-                    f"Failed to send evaluation status. Response: {json.dumps(resp_json)}"
-                )
-            except Exception:
-                raise ValueError(
-                    f"Failed to send evaluation status. Error: {response.text}"
-                )
-        return response
+            raise ValueError(
+                f"Failed to update evaluation status {evaluation_id}. "
+                f"Response: {response.text}"
+            )
+
+        return UpdateEvaluationResponse.model_validate(response.json())
 
     @classmethod
     def _headers(cls):

lmnr/sdk/types.py

@@ -2,7 +2,7 @@ import datetime
 import requests
 import pydantic
 import uuid
-from typing import Any, Literal, Optional, TypeAlias, Union
+from typing import Any, Awaitable, Callable, Literal, Optional, Union
 
 from .utils import to_dict
 
@@ -17,9 +17,9 @@ class ConditionedValue(pydantic.BaseModel):
     value: "NodeInput"
 
 
-Numeric: TypeAlias = Union[int, float]
-NodeInput: TypeAlias = Union[str, list[ChatMessage], ConditionedValue, Numeric, bool]
-PipelineOutput: TypeAlias = Union[NodeInput]
+Numeric = Union[int, float]
+NodeInput = Union[str, list[ChatMessage], ConditionedValue, Numeric, bool]
+PipelineOutput = Union[NodeInput]
 
 
 class PipelineRunRequest(pydantic.BaseModel):
@@ -74,12 +74,12 @@ class PipelineRunError(Exception):
             return super().__str__()
 
 
-EvaluationDatapointData: TypeAlias = dict[str, Any]
-EvaluationDatapointTarget: TypeAlias = dict[str, Any]
+EvaluationDatapointData = dict[str, Any]
+EvaluationDatapointTarget = dict[str, Any]
 
 
 # EvaluationDatapoint is a single data point in the evaluation
-class EvaluationDatapoint(pydantic.BaseModel):
+class Datapoint(pydantic.BaseModel):
     # input to the executor function. Must be a dict with string keys
     data: EvaluationDatapointData
     # input to the evaluator function (alongside the executor output).
@@ -87,24 +87,24 @@ class EvaluationDatapoint(pydantic.BaseModel):
     target: EvaluationDatapointTarget
 
 
-ExecutorFunctionReturnType: TypeAlias = Any
-EvaluatorFunctionReturnType: TypeAlias = Union[Numeric, dict[str, Numeric]]
+ExecutorFunctionReturnType = Any
+EvaluatorFunctionReturnType = Union[Numeric, dict[str, Numeric]]
 
-# ExecutorFunction: TypeAlias = Callable[
-#     [EvaluationDatapointData, *tuple[Any, ...], dict[str, Any]],
-#     Union[ExecutorFunctionReturnType, Awaitable[ExecutorFunctionReturnType]],
-# ]
+ExecutorFunction = Callable[
+    [EvaluationDatapointData, Any, dict[str, Any]],
+    Union[ExecutorFunctionReturnType, Awaitable[ExecutorFunctionReturnType]],
+]
 
 # EvaluatorFunction is a function that takes the output of the executor and the
 # target data, and returns a score. The score can be a single number or a
 # record of string keys and number values. The latter is useful for evaluating
 # multiple criteria in one go instead of running multiple evaluators.
-# EvaluatorFunction: TypeAlias = Callable[
-#     [ExecutorFunctionReturnType, *tuple[Any, ...], dict[str, Any]],
-#     Union[EvaluatorFunctionReturnType, Awaitable[EvaluatorFunctionReturnType]],
-# ]
+EvaluatorFunction = Callable[
+    [ExecutorFunctionReturnType, Any, dict[str, Any]],
+    Union[EvaluatorFunctionReturnType, Awaitable[EvaluatorFunctionReturnType]],
+]
 
-EvaluationStatus: TypeAlias = Literal["Started", "Finished", "Error"]
+EvaluationStatus = Literal["Started", "Finished", "Error"]
 
 
 class CreateEvaluationResponse(pydantic.BaseModel):
@@ -114,10 +114,14 @@ class CreateEvaluationResponse(pydantic.BaseModel):
     status: EvaluationStatus
     projectId: uuid.UUID
     metadata: Optional[dict[str, Any]] = None
+    averageScores: Optional[dict[str, Numeric]] = None
+
+
+UpdateEvaluationResponse = CreateEvaluationResponse
 
 
 class EvaluationResultDatapoint(pydantic.BaseModel):
     data: EvaluationDatapointData
     target: EvaluationDatapointTarget
-    executor_output: ExecutorFunctionReturnType
+    executorOutput: ExecutorFunctionReturnType
     scores: dict[str, Numeric]

lmnr/traceloop_sdk/__init__.py

@@ -55,9 +55,6 @@ class Traceloop:
 
         enable_content_tracing = is_content_tracing_enabled()
 
-        if exporter or processor:
-            print(Fore.GREEN + "Laminar exporting traces to a custom exporter")
-
         headers = os.getenv("TRACELOOP_HEADERS") or headers
 
         if isinstance(headers, str):
@@ -78,17 +75,7 @@ class Traceloop:
             print(Fore.RESET)
             return
 
-        if not exporter and not processor and headers:
-            print(
-                Fore.GREEN
-                + f"Laminar exporting traces to {api_endpoint}, authenticating with custom headers"
-            )
-
         if api_key and not exporter and not processor and not headers:
-            print(
-                Fore.GREEN
-                + f"Laminar exporting traces to {api_endpoint} authenticating with bearer token"
-            )
             headers = {
                 "Authorization": f"Bearer {api_key}",
             }

lmnr/traceloop_sdk/instruments.py

@@ -2,6 +2,8 @@ from enum import Enum
 
 
 class Instruments(Enum):
+    # The list of libraries which will be autoinstrumented
+    # if no specific instruments are provided to initialize()
     OPENAI = "openai"
     ANTHROPIC = "anthropic"
     COHERE = "cohere"
@@ -15,10 +17,6 @@ class Instruments(Enum):
     MILVUS = "milvus"
     TRANSFORMERS = "transformers"
     TOGETHER = "together"
-    REDIS = "redis"
-    REQUESTS = "requests"
-    URLLIB3 = "urllib3"
-    PYMYSQL = "pymysql"
     BEDROCK = "bedrock"
     REPLICATE = "replicate"
     VERTEXAI = "vertexai"
@@ -27,3 +25,10 @@ class Instruments(Enum):
     ALEPHALPHA = "alephalpha"
     MARQO = "marqo"
     LANCEDB = "lancedb"
+
+    # The following libraries will not be autoinstrumented unless
+    # specified explicitly in the initialize() call.
+    REDIS = "redis"
+    REQUESTS = "requests"
+    URLLIB3 = "urllib3"
+    PYMYSQL = "pymysql"

lmnr/traceloop_sdk/tracing/tracing.py

@@ -124,46 +124,34 @@ class TracerWrapper(object):
             # this makes sure otel context is propagated so we always want it
             ThreadingInstrumentor().instrument()
 
-            instrument_set = False
             if instruments is None:
                 init_instrumentations(should_enrich_metrics)
-                instrument_set = True
             else:
                 for instrument in instruments:
                     if instrument == Instruments.OPENAI:
                         if not init_openai_instrumentor(should_enrich_metrics):
                             print(Fore.RED + "Warning: OpenAI library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.ANTHROPIC:
                         if not init_anthropic_instrumentor(should_enrich_metrics):
                             print(
                                 Fore.RED + "Warning: Anthropic library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.COHERE:
                         if not init_cohere_instrumentor():
                             print(Fore.RED + "Warning: Cohere library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.PINECONE:
                         if not init_pinecone_instrumentor():
                             print(
                                 Fore.RED + "Warning: Pinecone library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.CHROMA:
                         if not init_chroma_instrumentor():
                             print(Fore.RED + "Warning: Chroma library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.GOOGLE_GENERATIVEAI:
                         if not init_google_generativeai_instrumentor():
                             print(
@@ -171,44 +159,32 @@ class TracerWrapper(object):
                                 + "Warning: Google Generative AI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.LANGCHAIN:
                         if not init_langchain_instrumentor():
                             print(
                                 Fore.RED + "Warning: LangChain library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.MISTRAL:
                         if not init_mistralai_instrumentor():
                             print(
                                 Fore.RED + "Warning: MistralAI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.OLLAMA:
                         if not init_ollama_instrumentor():
                             print(Fore.RED + "Warning: Ollama library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.LLAMA_INDEX:
                         if not init_llama_index_instrumentor():
                             print(
                                 Fore.RED + "Warning: LlamaIndex library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.MILVUS:
                         if not init_milvus_instrumentor():
                             print(Fore.RED + "Warning: Milvus library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.TRANSFORMERS:
                         if not init_transformers_instrumentor():
                             print(
@@ -216,72 +192,52 @@ class TracerWrapper(object):
                                 + "Warning: Transformers library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.TOGETHER:
                         if not init_together_instrumentor():
                             print(
                                 Fore.RED + "Warning: TogetherAI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.REQUESTS:
                         if not init_requests_instrumentor():
                             print(
                                 Fore.RED + "Warning: Requests library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.URLLIB3:
                         if not init_urllib3_instrumentor():
                             print(Fore.RED + "Warning: urllib3 library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.PYMYSQL:
                         if not init_pymysql_instrumentor():
                             print(Fore.RED + "Warning: PyMySQL library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.BEDROCK:
                         if not init_bedrock_instrumentor(should_enrich_metrics):
                             print(Fore.RED + "Warning: Bedrock library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.REPLICATE:
                         if not init_replicate_instrumentor():
                             print(
                                 Fore.RED + "Warning: Replicate library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.VERTEXAI:
                         if not init_vertexai_instrumentor():
                             print(
                                 Fore.RED + "Warning: Vertex AI library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.WATSONX:
                         if not init_watsonx_instrumentor():
                             print(Fore.RED + "Warning: Watsonx library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.WEAVIATE:
                         if not init_weaviate_instrumentor():
                             print(
                                 Fore.RED + "Warning: Weaviate library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.ALEPHALPHA:
                         if not init_alephalpha_instrumentor():
                             print(
@@ -289,26 +245,18 @@ class TracerWrapper(object):
                                 + "Warning: Aleph Alpha library does not exist."
                             )
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.MARQO:
                         if not init_marqo_instrumentor():
                             print(Fore.RED + "Warning: marqo library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.LANCEDB:
                         if not init_lancedb_instrumentor():
                             print(Fore.RED + "Warning: LanceDB library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
                     elif instrument == Instruments.REDIS:
                         if not init_redis_instrumentor():
                             print(Fore.RED + "Warning: redis library does not exist.")
                             print(Fore.RESET)
-                        else:
-                            instrument_set = True
 
                     else:
                         print(
@@ -324,13 +272,6 @@ class TracerWrapper(object):
                         )
                         print(Fore.RESET)
 
-            if not instrument_set:
-                print(
-                    Fore.RED + "Warning: No valid instruments set. Remove 'instrument' "
-                    "argument to use all instruments, or set a valid instrument."
-                )
-                print(Fore.RESET)
-
             obj.__content_allow_list = ContentAllowList()
 
             # Force flushes for debug environments (e.g. local development)
@@ -531,10 +472,6 @@ def init_instrumentations(should_enrich_metrics: bool):
     init_milvus_instrumentor()
     init_transformers_instrumentor()
     init_together_instrumentor()
-    init_redis_instrumentor()
-    init_requests_instrumentor()
-    init_urllib3_instrumentor()
-    init_pymysql_instrumentor()
     init_bedrock_instrumentor(should_enrich_metrics)
     init_replicate_instrumentor()
     init_vertexai_instrumentor()
@@ -545,6 +482,12 @@ def init_instrumentations(should_enrich_metrics: bool):
     init_lancedb_instrumentor()
     init_groq_instrumentor()
 
+    # These libraries are not instrumented by default, but if the user wants, he can manually specify them
+    # init_redis_instrumentor()
+    # init_requests_instrumentor()
+    # init_urllib3_instrumentor()
+    # init_pymysql_instrumentor()
+
 
 def init_openai_instrumentor(should_enrich_metrics: bool):
     try:

{lmnr-0.4.10.dist-info → lmnr-0.4.12b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.10
+Version: 0.4.12b1
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: argparse (>=1.0,<2.0)
 Requires-Dist: asyncio (>=3.0,<4.0)
 Requires-Dist: backoff (>=2.0,<3.0)
 Requires-Dist: colorama (>=0.4,<0.5)
@@ -54,6 +55,7 @@ Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: python-dotenv (>=1.0,<2.0)
 Requires-Dist: requests (>=2.0,<3.0)
 Requires-Dist: tenacity (>=8.0,<9.0)
+Requires-Dist: tqdm (>=4.0,<5.0)
 Description-Content-Type: text/markdown
 
 # Laminar Python
@@ -67,6 +69,9 @@ OpenTelemetry log sender for [Laminar](https://github.com/lmnr-ai/lmnr) for Pyth
 
 
 ## Quickstart
+
+First, install the package:
+
 ```sh
 python3 -m venv .myenv
 source .myenv/bin/activate  # or use your favorite env management tool
@@ -74,22 +79,39 @@ source .myenv/bin/activate  # or use your favorite env management tool
 pip install lmnr
 ```
 
-And the in your main Python file
+Then, you can initialize Laminar in your main file and instrument your code.
 
 ```python
-from lmnr import Laminar as L, Instruments
+import os
+from openai import OpenAI
+from lmnr import Laminar as L
 
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
-```
+L.initialize(
+    project_api_key=os.environ["LMNR_PROJECT_API_KEY"],
+)
 
-If you want to automatically instrument particular LLM, Vector DB, and related
-calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+def poem_writer(topic: str):
+    prompt = f"write a poem about {topic}"
+
+    # OpenAI calls are automatically instrumented
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": prompt},
+        ],
+    )
+    poem = response.choices[0].message.content
+    return poem
 
-You can pass an empty set as `instruments=set()` to disable any kind of automatic instrumentation. 
-Also if you want to automatically instrument all supported libraries, then pass `instruments=None` or don't pass `instruments` at all.
+if __name__ == "__main__":
+    print(poem_writer("laminar flow"))
+
+```
 
-Our code is based on the [OpenLLMetry](https://github.com/traceloop/openllmetry), open-source package
-by TraceLoop. Also, we are grateful to Traceloop for implementing autoinstrumentations for many libraries.
+Note that you need to only initialize Laminar once in your application.
 
 ### Project API key
 
@@ -98,67 +120,84 @@ You can either pass it to `.initialize()` or set it to `.env` at the root of you
 
 ## Instrumentation
 
-In addition to automatic instrumentation, we provide a simple `@observe()` decorator, if you want more fine-grained tracing
-or to trace other functions.
+### Manual instrumentation
 
-### Example
+To instrument any function in your code, we provide a simple `@observe()` decorator.
+This can be useful if you want to trace a request handler or a function which combines multiple LLM calls.
 
 ```python
 import os
 from openai import OpenAI
+from lmnr import Laminar as L, Instruments
 
-
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])
 
 client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
-@observe()  # annotate all functions you want to trace
-def poem_writer(topic="turbulence"):
+def poem_writer(topic: str):
     prompt = f"write a poem about {topic}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt},
+    ]
+
+    # OpenAI calls are still automatically instrumented
     response = client.chat.completions.create(
         model="gpt-4o",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": prompt},
-        ],
+        messages=messages,
     )
     poem = response.choices[0].message.content
+
     return poem
 
-print(poem_writer(topic="laminar flow"))
+@observe()
+def generate_poems():
+    poem1 = poem_writer(topic="laminar flow")
+    L.event("is_poem_generated", True)
+    poem2 = poem_writer(topic="turbulence")
+    L.event("is_poem_generated", True)
+    poems = f"{poem1}\n\n---\n\n{poem2}"
+    return poems
 ```
 
-### Manual instrumentation
-
-Also, you can `Laminar.start_as_current_span` if you want to record a chunk of your code.
+Also, you can use `Laminar.start_as_current_span` if you want to record a chunk of your code using `with` statement.
 
 ```python
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+def handle_user_request(topic: str):
+    with L.start_as_current_span(name="poem_writer", input=topic):
+        ...
+
+        poem = poem_writer(topic=topic)
+        
+        ...
+        
+        # while within the span, you can attach laminar events to it
+        L.event("is_poem_generated", True)
 
-def poem_writer(topic="turbulence"):
-    prompt = f"write a poem about {topic}"
-    messages = [
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": prompt},
-    ]
+        # Use set_span_output to record the output of the span
+        L.set_span_output(poem)
+```
 
-    with L.start_as_current_span(name="poem_writer", input=messages):
-        # OpenAI calls are still automatically instrumented with OpenLLMetry
-        response = client.chat.completions.create(
-            model="gpt-4o",
-            messages=messages,
-        )
-        poem = response.choices[0].message.content
-        # while within the span, you can attach laminar events to it
-        L.event("event_name", "event_value")
+### Automatic instrumentation
 
-        L.set_span_output(poem) # set an output
+Laminar allows you to automatically instrument majority of the most popular LLM, Vector DB, database, requests, and other libraries.
 
-        return poem
+If you want to automatically instrument a default set of libraries, then simply do NOT pass `instruments` argument to `.initialize()`.
+See the full list of available instrumentations in the [enum](/src/lmnr/traceloop_sdk/instruments.py).
+
+If you want to automatically instrument only specific LLM, Vector DB, or other
+calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+For example, if you want to only instrument OpenAI and Anthropic, then do the following:
+
+```python
+from lmnr import Laminar as L, Instruments
+
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"], instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
 ```
 
+If you want to fully disable any kind of autoinstrumentation, pass an empty set as `instruments=set()` to `.initialize()`. 
+
+Majority of the autoinstrumentations are provided by Traceloop's [OpenLLMetry](https://github.com/traceloop/openllmetry).
 
 ## Sending events
 

{lmnr-0.4.10.dist-info → lmnr-0.4.12b1.dist-info}/RECORD

@@ -1,18 +1,19 @@
-lmnr/__init__.py,sha256=bA1f7JsEdSdU93HTz3SQLSanq-UgZGvb5I2OE0CWGR8,233
+lmnr/__init__.py,sha256=5Ks8UIicCzCBgwSz0MOX3I7jVruPMUO3SmxIwUoODzQ,231
+lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-lmnr/sdk/decorators.py,sha256=W46diLcINe0HAhxktrjbfQnaIfklSb0AydBHHxiko9U,2314
-lmnr/sdk/evaluations.py,sha256=EaRcwbdXxj4w2yzak1xFv-YhDuxRVentQcJ-CypBoH0,6307
-lmnr/sdk/laminar.py,sha256=M8HdP6ZYJHdngUVrGj4GMZxz_EZyx3woHm-UpfWmIvs,18439
+lmnr/sdk/decorators.py,sha256=O8S4PI6LUfdWPkbroigl5khtnkyhp24J8qzSdlvCs44,2227
+lmnr/sdk/evaluations.py,sha256=kIME5Ahc_N3hVr7CXuG72oLDl7PjMg2JJlaBSkCG_S0,9999
+lmnr/sdk/laminar.py,sha256=r9jTmIg4ljPlMiy6RVkmldd2i8l4p16ixmvVwyI8jOE,19001
 lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
-lmnr/sdk/types.py,sha256=w7BJsoEPHiNps62cQt3Hd6tEZ7ZFCKRTPzcwdD6rNak,4050
+lmnr/sdk/types.py,sha256=NKXohmXewppXUt_AwkTyZpohH6n4IStsVeEGILTB8e0,4006
 lmnr/sdk/utils.py,sha256=ZsGJ86tq8lIbvOhSb1gJWH5K3GylO_lgX68FN6rG2nM,3358
 lmnr/traceloop_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/traceloop_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
-lmnr/traceloop_sdk/__init__.py,sha256=J-zVw6j0DmceVvJVZXAFcCzN_scz9hB3X17NQgPMgOg,4420
+lmnr/traceloop_sdk/__init__.py,sha256=-wa25NtU7BeTgqjT6rpVIEdaVP4aOjLgbYrc3B3-JaM,3932
 lmnr/traceloop_sdk/config/__init__.py,sha256=EGN3ixOt_ORbMxqaQdLaC14kmO-gyG4mnGJ2GfN-R-E,364
 lmnr/traceloop_sdk/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/traceloop_sdk/decorators/base.py,sha256=wcqXF0iVQgRXMyWTcJ5QvL_6q2y_gttwsX8dllmAtWM,4891
-lmnr/traceloop_sdk/instruments.py,sha256=G5EFAbpc20WD3M6xK6rlbj-Yy_r_f1m3gidY6UXzSRQ,701
+lmnr/traceloop_sdk/instruments.py,sha256=oMvIASueW3GeChpjIdH-DD9aFBVB8OtHZ0HawppTrlI,942
 lmnr/traceloop_sdk/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/traceloop_sdk/metrics/metrics.py,sha256=AlQ2a2os1WcZbfBd155u_UzBbPrbuPia6O_HbojV9Wc,5055
 lmnr/traceloop_sdk/tests/__init__.py,sha256=RYnG0-8zbXL0-2Ste1mEBf5sN4d_rQjGTCgPBuaZC74,20
@@ -39,14 +40,14 @@ lmnr/traceloop_sdk/tracing/__init__.py,sha256=Ckq7zCM26VdJVB5tIZv0GTPyMZKyfso_KW
 lmnr/traceloop_sdk/tracing/content_allow_list.py,sha256=3feztm6PBWNelc8pAZUcQyEGyeSpNiVKjOaDk65l2ps,846
 lmnr/traceloop_sdk/tracing/context_manager.py,sha256=csVlB6kDmbgSPsROHwnddvGGblx55v6lJMRj0wsSMQM,304
 lmnr/traceloop_sdk/tracing/manual.py,sha256=RPwEreHHdzmw7g15u4G21GqhHOvRp7d72ylQNLG1jRM,1841
-lmnr/traceloop_sdk/tracing/tracing.py,sha256=VFrf5D6CC3DquLy_19_5I_L_w1kO2X61KvPW0XD26-k,42347
+lmnr/traceloop_sdk/tracing/tracing.py,sha256=5e8AsiFKaIO6zqAbMfhw242glVsQUkxbNhTWP7QDqSg,40108
 lmnr/traceloop_sdk/utils/__init__.py,sha256=pNhf0G3vTd5ccoc03i1MXDbricSaiqCbi1DLWhSekK8,604
 lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQvzJk_fZH0OOsRAM1iZQXsR8,2112
 lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
 lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
 lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
-lmnr-0.4.10.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.10.dist-info/METADATA,sha256=VPXsfYwAy1uTv_qucCqgAmyGyZMngr0HuysSSdpX8Jw,10999
-lmnr-0.4.10.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lmnr-0.4.10.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
-lmnr-0.4.10.dist-info/RECORD,,
+lmnr-0.4.12b1.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.12b1.dist-info/METADATA,sha256=6ne6QRKpBg09-X6ypcHSgqfbwjLvmxPAdknTPfA00F0,11992
+lmnr-0.4.12b1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lmnr-0.4.12b1.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.12b1.dist-info/RECORD,,

lmnr-0.4.12b1.dist-info/entry_points.txt

@@ -0,0 +1,3 @@
+[console_scripts]
+lmnr=lmnr.cli:cli
+

lmnr-0.4.10.dist-info/entry_points.txt

@@ -1,3 +0,0 @@
-[console_scripts]
-lmnr=lmnr.cli.cli:cli
-