lmnr 0.4.17__py3-none-any.whl → 0.4.18b0__py3-none-any.whl

lmnr/sdk/evaluations.py

@@ -79,9 +79,13 @@ class EvaluationReporter:
         self.cli_progress.close()
         sys.stderr.write(f"\nError: {error}\n")
 
-    def stop(self, average_scores: dict[str, Numeric], project_id: str, evaluation_id: str):
+    def stop(
+        self, average_scores: dict[str, Numeric], project_id: str, evaluation_id: str
+    ):
         self.cli_progress.close()
-        print(f"\nCheck progress and results at {get_evaluation_url(project_id, evaluation_id)}\n")
+        print(
+            f"\nCheck progress and results at {get_evaluation_url(project_id, evaluation_id)}\n"
+        )
         print("Average scores:")
         for name, score in average_scores.items():
             print(f"{name}: {score}")
@@ -124,35 +128,41 @@ class Evaluation:
         Initializes an instance of the Evaluations class.
 
         Parameters:
-            data (Union[List[Union[EvaluationDatapoint, dict]], EvaluationDataset]): List of data points to evaluate or an evaluation dataset.
+            data (Union[List[EvaluationDatapoint|dict], EvaluationDataset]):\
+                List of data points to evaluate or an evaluation dataset.
                             `data` is the input to the executor function,
                             `target` is the input to the evaluator function.
-            executor (Callable[..., Any]): The executor function.
-                            Takes the data point + any additional arguments
+            executor (Callable[..., Any]): The executor function.\
+                            Takes the data point + any additional arguments\
                             and returns the output to evaluate.
-            evaluators (List[Callable[..., Any]]): List of evaluator functions.
-                Each evaluator function takes the output of the executor _and_
-                the target data, and returns a score. The score can be a
-                single number or a record of string keys and number values.
-                If the score is a single number, it will be named after the
-                evaluator function. If the function is anonymous, it will be
-                named `evaluator_${index}`, where index is the index of the
+            evaluators (List[Callable[..., Any]]): List of evaluator functions.\
+                Each evaluator function takes the output of the executor _and_\
+                the target data, and returns a score. The score can be a\
+                single number or a record of string keys and number values.\
+                If the score is a single number, it will be named after the\
+                evaluator function. If the function is anonymous, it will be\
+                named `evaluator_${index}`, where index is the index of the\
                 evaluator function in the list starting from 1.
             group_id (Optional[str], optional): Group id of the evaluation.
                             Defaults to "default".
-            name (Optional[str], optional): The name of the evaluation.
+            name (Optional[str], optional): The name of the evaluation.\
                             It will be auto-generated if not provided.
             batch_size (int, optional): The batch size for evaluation.
                             Defaults to DEFAULT_BATCH_SIZE.
             project_api_key (Optional[str], optional): The project API key.
                             Defaults to an empty string.
-            base_url (Optional[str], optional): The base URL for the Laminar API.
-                            Useful if self-hosted elsewhere.
+            base_url (Optional[str], optional): The base URL for Laminar API.\
+                            Useful if self-hosted elsewhere. Do NOT include the\
+                            port, use `http_port` and `grpc_port` instead.
                             Defaults to "https://api.lmnr.ai".
-            http_port (Optional[int], optional): The port for the Laminar API HTTP service.
-                            Defaults to 443.
-            instruments (Optional[Set[Instruments]], optional): Set of modules to auto-instrument.
-                            Defaults to None. If None, all available instruments will be used.
+            http_port (Optional[int], optional): The port for Laminar API\
+                            HTTP service. Defaults to 443 if not specified.
+            grpc_port (Optional[int], optional): The port for Laminar API\
+                            gRPC service. Defaults to 8443 if not specified.
+            instruments (Optional[Set[Instruments]], optional): Set of modules\
+                to auto-instrument. If None, all available instruments will be\
+                used.
+                Defaults to None.
         """
 
         if not evaluators:
@@ -160,8 +170,12 @@ class Evaluation:
 
         # TODO: Compile regex once and then reuse it
         for evaluator_name in evaluators:
-            if not re.match(r'^[\w\s-]+$', evaluator_name):
-                raise ValueError(f'Invalid evaluator key: "{evaluator_name}". Keys must only contain letters, digits, hyphens, underscores, or spaces.')
+            if not re.match(r"^[\w\s-]+$", evaluator_name):
+                raise ValueError(
+                    f'Invalid evaluator key: "{evaluator_name}". '
+                    "Keys must only contain letters, digits, hyphens,"
+                    "underscores, or spaces."
+                )
 
         self.is_finished = False
         self.reporter = EvaluationReporter()
@@ -207,7 +221,9 @@ class Evaluation:
             self.is_finished = True
             return
         else:
-            evaluation = L.create_evaluation(data=result_datapoints, group_id=self.group_id, name=self.name)
+            evaluation = L.create_evaluation(
+                data=result_datapoints, group_id=self.group_id, name=self.name
+            )
             average_scores = get_average_scores(result_datapoints)
             self.reporter.stop(average_scores, evaluation.projectId, evaluation.id)
             self.is_finished = True
@@ -216,7 +232,7 @@ class Evaluation:
         result_datapoints = []
         for i in range(0, len(self.data), self.batch_size):
             batch = (
-                self.data[i: i + self.batch_size]
+                self.data[i : i + self.batch_size]
                 if isinstance(self.data, list)
                 else self.data.slice(i, i + self.batch_size)
             )
@@ -294,43 +310,58 @@ def evaluate(
     instruments: Optional[Set[Instruments]] = None,
 ) -> Optional[Awaitable[None]]:
     """
-    If added to the file which is called through lmnr eval command, then simply registers the evaluation.
-    Otherwise, if there is no event loop, creates it and runs the evaluation until completion.
-    If there is an event loop, schedules the evaluation as a task in the event loop and returns an awaitable handle.
+    If added to the file which is called through `lmnr eval` command, then
+    registers the evaluation; otherwise, runs the evaluation.
+
+    If there is no event loop, creates it and runs the evaluation until
+    completion.
+    If there is an event loop, schedules the evaluation as a task in the
+    event loop and returns an awaitable handle.
 
     Parameters:
-        data (Union[List[Union[EvaluationDatapoint, dict]], EvaluationDataset]): List of data points to evaluate or an evaluation dataset.
+        data (Union[list[EvaluationDatapoint|dict]], EvaluationDataset]):\
+                    List of data points to evaluate or an evaluation dataset.
                         `data` is the input to the executor function,
                         `target` is the input to the evaluator function.
-        executor (Callable[..., Any]): The executor function.
-                        Takes the data point + any additional arguments
+        executor (Callable[..., Any]): The executor function.\
+                        Takes the data point + any additional arguments\
                         and returns the output to evaluate.
-        evaluators (List[Callable[..., Any]]): List of evaluator functions.
-            Each evaluator function takes the output of the executor _and_
-            the target data, and returns a score. The score can be a
-            single number or a record of string keys and number values.
-            If the score is a single number, it will be named after the
-            evaluator function. If the function is anonymous, it will be
-            named `evaluator_${index}`, where index is the index of the
+        evaluators (List[Callable[..., Any]]): List of evaluator functions.\
+            Each evaluator function takes the output of the executor _and_\
+            the target data, and returns a score. The score can be a\
+            single number or a record of string keys and number values.\
+            If the score is a single number, it will be named after the\
+            evaluator function. If the function is anonymous, it will be\
+            named `evaluator_${index}`, where index is the index of the\
             evaluator function in the list starting from 1.
-        group_id (Optional[str], optional): Group name which is same
-                        as the feature you are evaluating in your project or application.
-                        Defaults to "default".
-        name (Optional[str], optional): Optional name of the evaluation. Used to easily
-                        identify the evaluation in the group.
+        group_id (Optional[str], optional): an identifier to group evaluations.\
+                        It is practical to group evaluations that evaluate\
+                        the same feature on the same dataset, to be able to\
+                        view their comparisons in the same place. If not\
+                        provided, defaults to "default".
+                        Defaults to None
+        name (Optional[str], optional): Optional name of the evaluation.\
+                        Used to identify the evaluation in the group.\
+                        If not provided, a random name will be generated.
+                        Defaults to None.
         batch_size (int, optional): The batch size for evaluation.
                         Defaults to DEFAULT_BATCH_SIZE.
         project_api_key (Optional[str], optional): The project API key.
-                        Defaults to an empty string.
-        base_url (Optional[str], optional): The base URL for the Laminar API.
-                        Useful if self-hosted elsewhere.
+                        Defaults to None.
+        base_url (Optional[str], optional): The base URL for Laminar API.\
+                        Useful if self-hosted elsewhere. Do NOT include the\
+                        port, use `http_port` and `grpc_port` instead.
                         Defaults to "https://api.lmnr.ai".
-        http_port (Optional[int], optional): The port for the Laminar API HTTP service.
-                        Defaults to 443.
-        grpc_port (Optional[int], optional): The port for the Laminar API gRPC service.
-                        Defaults to 8443.
-        instruments (Optional[Set[Instruments]], optional): Set of modules to auto-instrument.
-                        Defaults to None. If None, all available instruments will be used.
+        http_port (Optional[int], optional): The port for Laminar API's HTTP\
+                        service. 443 is used if not specified.
+                        Defaults to None.
+        grpc_port (Optional[int], optional): The port for Laminar API's gRPC\
+                        service. 8443 is used if not specified.
+                        Defaults to None.
+        instruments (Optional[Set[Instruments]], optional): Set of modules to\
+                        auto-instrument. If None, all available instruments\
+                        will be used.
+                        Defaults to None.
     """
 
     evaluation = Evaluation(

lmnr/sdk/laminar.py

@@ -1,3 +1,4 @@
+import re
 from lmnr.traceloop_sdk.instruments import Instruments
 from opentelemetry import context
 from opentelemetry.trace import (
@@ -72,23 +73,23 @@ class Laminar:
         decorators.
 
         Args:
-            project_api_key (Optional[str], optional): Laminar project api key.
-                            You can generate one by going to the projects
-                            settings page on the Laminar dashboard.
-                            If not specified, it will try to read from the
-                            LMNR_PROJECT_API_KEY environment variable
+            project_api_key (Optional[str], optional): Laminar project api key.\
+                            You can generate one by going to the projects\
+                            settings page on the Laminar dashboard.\
+                            If not specified, it will try to read from the\
+                            LMNR_PROJECT_API_KEY environment variable\
                             in os.environ or in .env file.
                             Defaults to None.
-            env (dict[str, str], optional): Default environment passed to
-                            `run` requests, unless
-                            overriden at request time. Usually, model
-                            provider keys are stored here.
+            env (dict[str, str], optional): Default environment passed to\
+                            `run` requests, unless overriden at request time.\
+                            Usually, model provider keys are stored here.
                             Defaults to {}.
-            base_url (Optional[str], optional): Laminar API url.
+            base_url (Optional[str], optional): Laminar API url. Do NOT include\
+                            the port number, use `http_port` and `grpc_port`.\
                             If not specified, defaults to https://api.lmnr.ai.
-            http_port (Optional[int], optional): Laminar API http port.
+            http_port (Optional[int], optional): Laminar API http port.\
                             If not specified, defaults to 443.
-            grpc_port (Optional[int], optional): Laminar API grpc port.
+            grpc_port (Optional[int], optional): Laminar API grpc port.\
                             If not specified, defaults to 8443.
 
         Raises:
@@ -108,9 +109,14 @@ class Laminar:
                 " your project API key or set the LMNR_PROJECT_API_KEY"
                 " environment variable in your environment or .env file"
             )
-
-        cls.__base_http_url = f"{base_url or 'https://api.lmnr.ai'}:{http_port or 443}"
-        cls.__base_grpc_url = f"{base_url or 'https://api.lmnr.ai'}:{grpc_port or 8443}"
+        if re.search(r":\d{1,5}$", base_url):
+            raise ValueError(
+                "Please provide the `base_url` without the port number. "
+                "Use the `http_port` and `grpc_port` arguments instead."
+            )
+        url = base_url or "https://api.lmnr.ai"
+        cls.__base_http_url = f"{url}:{http_port or 443}"
+        cls.__base_grpc_url = f"{url}:{grpc_port or 8443}"
 
         cls.__env = env
         cls.__initialized = True
@@ -153,22 +159,22 @@ class Laminar:
         """Runs the pipeline with the given inputs
 
         Args:
-            pipeline (str): name of the Laminar pipeline.
+            pipeline (str): name of the Laminar pipeline.\
                 The pipeline must have a target version set.
             inputs (dict[str, NodeInput]):
-                inputs to the endpoint's target pipeline.
+                inputs to the endpoint's target pipeline.\
                 Keys in the dictionary must match input node names
             env (dict[str, str], optional):
                 Environment variables for the pipeline execution.
                 Defaults to {}.
             metadata (dict[str, str], optional):
-                any custom metadata to be stored
-                with execution trace. Defaults to {}.
-            parent_span_id (Optional[uuid.UUID], optional):
-                parent span id for the resulting span.
+                any custom metadata to be stored with execution trace.
+                Defaults to {}.
+            parent_span_id (Optional[uuid.UUID], optional): parent span id for\
+                the resulting span.
                 Defaults to None.
-            trace_id (Optional[uuid.UUID], optional):
-                trace id for the resulting trace.
+            trace_id (Optional[uuid.UUID], optional): trace id for the\
+                resulting trace.
                 Defaults to None.
 
         Returns:
@@ -195,7 +201,7 @@ class Laminar:
             request = PipelineRunRequest(
                 inputs=inputs,
                 pipeline=pipeline,
-                env=env,
+                env=env or cls.__env,
                 metadata=metadata,
                 parent_span_id=parent_span_id,
                 trace_id=trace_id,
@@ -228,21 +234,23 @@ class Laminar:
         value: Optional[AttributeValue] = None,
         timestamp: Optional[Union[datetime.datetime, int]] = None,
     ):
-        """Associate an event with the current span. If event with such name never
-        existed, Laminar will create a new event and infer its type from the value.
-        If the event already exists, Laminar will append the value to the event
-        if and only if the value is of a matching type. Otherwise, the event won't
-        be recorded Supported types are string, numeric, and boolean. If the value
+        """Associate an event with the current span. If event with such
+        name never existed, Laminar will create a new event and infer its type
+        from the value. If the event already exists, Laminar will append the
+        value to the event if and only if the value is of a matching type.
+        Otherwise, the event won't be recorded.
+        Supported types are string, numeric, and boolean. If the value
         is `None`, event is considered a boolean tag with the value of `True`.
 
         Args:
             name (str): event name
-            value (Optional[AttributeValue]): event value. Must be a primitive type.
-                            Boolean true is assumed in the backend if value is None.
+            value (Optional[AttributeValue]): event value. Must be a primitive\
+                            type. Boolean true is assumed in the backend if\
+                            `value` is None.
                             Defaults to None.
-            timestamp (Optional[Union[datetime.datetime, int]], optional):
-                            If int, must be epoch nanoseconds. If not
-                            specified, relies on the underlying OpenTelemetry
+            timestamp (Optional[Union[datetime.datetime, int]], optional):\
+                            If int, must be epoch nanoseconds. If not\
+                            specified, relies on the underlying OpenTelemetry\
                             implementation. Defaults to None.
         """
         if timestamp and isinstance(timestamp, datetime.datetime):
@@ -272,18 +280,18 @@ class Laminar:
         name: str,
         input: Any = None,
     ):
-        """Start a new span as the current span. Useful for manual instrumentation.
-        This is the preferred and more stable way to use manual instrumentation.
+        """Start a new span as the current span. Useful for manual
+        instrumentation.
 
         Usage example:
         ```python
-        with Laminar.start_as_current_span("my_span", input="my_input"):
+        with Laminar.start_as_current_span("my_span", input="my_input") as span:
             await my_async_function()
         ```
 
         Args:
             name (str): name of the span
-            input (Any, optional): input to the span. Will be sent as an
+            input (Any, optional): input to the span. Will be sent as an\
                 attribute, so must be json serializable. Defaults to None.
         """
         with get_tracer() as tracer:
@@ -310,10 +318,11 @@ class Laminar:
 
     @classmethod
     def set_span_output(cls, output: Any = None):
-        """Set the output of the current span. Useful for manual instrumentation.
+        """Set the output of the current span. Useful for manual
+        instrumentation.
 
         Args:
-            output (Any, optional): output of the span. Will be sent as an
+            output (Any, optional): output of the span. Will be sent as an\
                 attribute, so must be json serializable. Defaults to None.
         """
         span = get_current_span()
@@ -331,12 +340,12 @@ class Laminar:
         thread).
 
         Args:
-            session_id (Optional[str], optional): Custom session id.
-                            Useful to debug and group long-running
+            session_id (Optional[str], optional): Custom session id.\
+                            Useful to debug and group long-running\
                             sessions/conversations.
                             Defaults to None.
-            user_id (Optional[str], optional): Custom user id.
-                            Useful for grouping spans or traces by user.
+            user_id (Optional[str], optional): Custom user id.\
+                            Useful for grouping spans or traces by user.\
                             Defaults to None.
         """
         association_properties = {}

{lmnr-0.4.17.dist-info → lmnr-0.4.18b0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.17
+Version: 0.4.18b0
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -249,7 +249,7 @@ You can run evaluations locally by providing executor (part of the logic used in
 
 \* If you already have the outputs of executors you want to evaluate, you can specify the executor as an identity function, that takes in `data` and returns only needed value(s) from it.
 
-[Read docs](https://docs.lmnr.ai/evaluations/introduction) to learn more about evaluations.
+Read the [docs](https://docs.lmnr.ai/evaluations/introduction) to learn more about evaluations.
 
 ## Laminar pipelines as prompt chain managers
 

{lmnr-0.4.17.dist-info → lmnr-0.4.18b0.dist-info}/RECORD

@@ -2,8 +2,8 @@ lmnr/__init__.py,sha256=5Ks8UIicCzCBgwSz0MOX3I7jVruPMUO3SmxIwUoODzQ,231
 lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
 lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/sdk/decorators.py,sha256=ZSDaEZyjo-RUzRCltsNbe6x0t9SKl2xRQ2q4uaKvXtk,2250
-lmnr/sdk/evaluations.py,sha256=Tukl2pW_x13ittzG5XQpF1TweYo3fpD4eLInplQ4YYI,14152
-lmnr/sdk/laminar.py,sha256=-qvfWYUAT63pBSdN6LFj6oT7JUbqEGVRunu-5tckzaM,14698
+lmnr/sdk/evaluations.py,sha256=C2lhrevH3PDvnTX-qfop1k-NTqTFX4MWaSkc9oG5kxc,15199
+lmnr/sdk/laminar.py,sha256=Dy8RVgGIev4dTjnu-LSwFoI0Un09EoLJDv_Fx4q2vvQ,15018
 lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
 lmnr/sdk/types.py,sha256=HvaZEqVRduCZbkF7Cp8rgS5oBbc1qPvOD3PP9tFrRu4,4826
 lmnr/sdk/utils.py,sha256=s81p6uJehgJSaLWy3sR5fTpEDH7vzn3i_UujUHChl6M,3346
@@ -44,8 +44,8 @@ lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQv
 lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
 lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
 lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
-lmnr-0.4.17.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.17.dist-info/METADATA,sha256=CL8gLkuosznUmzBgtwoJolDYa_6a6CLgQztWpIOaTsE,10474
-lmnr-0.4.17.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-lmnr-0.4.17.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.17.dist-info/RECORD,,
+lmnr-0.4.18b0.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.18b0.dist-info/METADATA,sha256=IGISzLTkZUs6hMecDq-Pmg31C-hYQ3MguS6DRpsr0qQ,10480
+lmnr-0.4.18b0.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lmnr-0.4.18b0.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.18b0.dist-info/RECORD,,