lmnr 0.4.47__py3-none-any.whl → 0.4.49__py3-none-any.whl

lmnr/cli.py

@@ -1,40 +1,74 @@
 from argparse import ArgumentParser
 import asyncio
 import importlib.util
+import logging
 import os
+import re
 import sys
 
 from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
+from .sdk.log import ColorfulFormatter
+
+LOG = logging.getLogger(__name__)
+console_log_handler = logging.StreamHandler()
+console_log_handler.setFormatter(ColorfulFormatter())
+LOG.addHandler(console_log_handler)
+
+
+EVAL_DIR = "evals"
 
 
 async def run_evaluation(args):
     sys.path.append(os.getcwd())
 
-    prep_token = PREPARE_ONLY.set(True)
-    try:
-        file = os.path.abspath(args.file)
-        name = "user_module"
+    if args.file is None:
+        files = [
+            os.path.join(EVAL_DIR, f)
+            for f in os.listdir(EVAL_DIR)
+            if re.match(r".*_eval\.py$", f) or re.match(r"eval_.*\.py$", f)
+        ]
+        if len(files) == 0:
+            LOG.error("No evaluation files found in evals directory")
+            return
+        files.sort()
+        LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
+
+    else:
+        files = [args.file]
 
-        spec = importlib.util.spec_from_file_location(name, file)
-        if spec is None or spec.loader is None:
-            raise ImportError(f"Could not load module specification from {file}")
-        mod = importlib.util.module_from_spec(spec)
-        sys.modules[name] = mod
+    for file in files:
+        prep_token = PREPARE_ONLY.set(True)
+        LOG.info(f"Running evaluation from {file}")
+        try:
+            file = os.path.abspath(file)
+            name = "user_module" + file
 
-        spec.loader.exec_module(mod)
-        evaluation = EVALUATION_INSTANCE.get()
-        if evaluation is None:
-            raise RuntimeError("Evaluation instance not found")
+            spec = importlib.util.spec_from_file_location(name, file)
+            if spec is None or spec.loader is None:
+                LOG.error(f"Could not load module specification from {file}")
+                if args.fail_on_error:
+                    return
+                continue
+            mod = importlib.util.module_from_spec(spec)
+            sys.modules[name] = mod
 
-        await evaluation.run()
-    finally:
-        PREPARE_ONLY.reset(prep_token)
+            spec.loader.exec_module(mod)
+            evaluation = EVALUATION_INSTANCE.get()
+            if evaluation is None:
+                LOG.warning("Evaluation instance not found")
+                if args.fail_on_error:
+                    return
+                continue
+
+            await evaluation.run()
+        finally:
+            PREPARE_ONLY.reset(prep_token)
 
 
 def cli():
     parser = ArgumentParser(
         prog="lmnr",
-        description="CLI for Laminar",
+        description="CLI for Laminar. Call `lmnr [subcommand] --help` for more information on each subcommand.",
     )
 
     subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
@@ -44,7 +78,21 @@ def cli():
         description="Run an evaluation",
         help="Run an evaluation",
     )
-    parser_eval.add_argument("file", help="A file containing the evaluation to run")
+    parser_eval.add_argument(
+        "file",
+        nargs="?",
+        help="A file containing the evaluation to run."
+        + "If no file name is provided, all evaluation files in the `evals` directory are run as long"
+        + "as they match *_eval.py or eval_*.py",
+        default=None,
+    )
+
+    parser_eval.add_argument(
+        "--fail-on-error",
+        action="store_true",
+        default=False,
+        help="Fail on error",
+    )
 
     parsed = parser.parse_args()
     if parsed.subcommand == "eval":

lmnr/sdk/evaluations.py

@@ -29,8 +29,17 @@ from .utils import is_async
 DEFAULT_BATCH_SIZE = 5
 
 
-def get_evaluation_url(project_id: str, evaluation_id: str):
-    return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
+def get_evaluation_url(
+    project_id: str, evaluation_id: str, base_url: str = "https://www.lmnr.ai"
+):
+    url = base_url
+    if url.endswith("/"):
+        url = url[:-1]
+    if url.endswith("localhost") or url.endswith("127.0.0.1"):
+        # We best effort assume that the frontend is running on port 3000
+        # TODO: expose the frontend port?
+        url = url + ":3000"
+    return f"{url}/project/{project_id}/evaluations/{evaluation_id}"
 
 
 def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Numeric]:
@@ -49,8 +58,8 @@ def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Nu
 
 
 class EvaluationReporter:
-    def __init__(self):
-        pass
+    def __init__(self, base_url: str = "https://www.lmnr.ai"):
+        self.base_url = base_url
 
     def start(self, length: int):
         self.cli_progress = tqdm(
@@ -71,7 +80,7 @@ class EvaluationReporter:
     ):
         self.cli_progress.close()
         print(
-            f"\nCheck the results at {get_evaluation_url(project_id, evaluation_id)}\n"
+            f"\nCheck the results at {get_evaluation_url(project_id, evaluation_id, self.base_url)}\n"
         )
         print("Average scores:")
         for name, score in average_scores.items():
@@ -160,7 +169,7 @@ class Evaluation:
                 )
 
         self.is_finished = False
-        self.reporter = EvaluationReporter()
+        self.reporter = EvaluationReporter(base_url)
         if isinstance(data, list):
             self.data = [
                 (Datapoint.model_validate(point) if isinstance(point, dict) else point)

lmnr/sdk/laminar.py

@@ -12,6 +12,7 @@ from lmnr.openllmetry_sdk.tracing.attributes import (
 from lmnr.openllmetry_sdk.decorators.base import json_dumps
 from opentelemetry import context, trace
 from opentelemetry.context import attach, detach, set_value
+from opentelemetry.sdk.trace import SpanProcessor
 from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
 from opentelemetry.util.types import AttributeValue
 
@@ -79,6 +80,7 @@ class Laminar:
         http_port: Optional[int] = None,
         grpc_port: Optional[int] = None,
         instruments: Optional[Set[Instruments]] = None,
+        _processor: Optional[SpanProcessor] = None,
     ):
         """Initialize Laminar context across the application.
         This method must be called before using any other Laminar methods or
@@ -133,6 +135,7 @@ class Laminar:
         cls.__env = env
         cls.__initialized = True
         cls._initialize_logger()
+
         Traceloop.init(
             exporter=OTLPSpanExporter(
                 endpoint=cls.__base_grpc_url,
@@ -268,13 +271,9 @@ class Laminar:
         value: Optional[AttributeValue] = None,
         timestamp: Optional[Union[datetime.datetime, int]] = None,
     ):
-        """Associate an event with the current span. If event with such
-        name never existed, Laminar will create a new event and infer its type
-        from the value. If the event already exists, Laminar will append the
-        value to the event if and only if the value is of a matching type.
-        Otherwise, the event won't be recorded.
-        Supported types are string, numeric, and boolean. If the value
-        is `None`, event is considered a boolean tag with the value of `True`.
+        """Associate an event with the current span. If using manual\
+        instrumentation, use raw OpenTelemetry `span.add_event()` instead.\
+       `value` will be saved as a `lmnr.event.value` attribute.
 
         Args:
             name (str): event name
@@ -400,7 +399,7 @@ class Laminar:
                     )
                 yield span
 
-            # # TODO: Figure out if this is necessary
+            # TODO: Figure out if this is necessary
             try:
                 detach(ctx_token)
             except Exception:
@@ -672,9 +671,9 @@ class Laminar:
     def clear_metadata(cls):
         """Clear the metadata from the context"""
         props: dict = copy.copy(context.get_value("association_properties"))
-        for k in props.keys():
-            if k.startswith("metadata."):
-                props.pop(k)
+        metadata_keys = [k for k in props.keys() if k.startswith("metadata.")]
+        for k in metadata_keys:
+            props.pop(k)
         set_association_properties(props)
 
     @classmethod

lmnr/sdk/log.py

@@ -24,6 +24,29 @@ class CustomFormatter(logging.Formatter):
         return formatter.format(record)
 
 
+class ColorfulFormatter(logging.Formatter):
+    grey = "\x1b[38;20m"
+    green = "\x1b[32;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    fmt = "Laminar %(levelname)s: %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: grey + fmt + reset,
+        logging.INFO: green + fmt + reset,
+        logging.WARNING: yellow + fmt + reset,
+        logging.ERROR: red + fmt + reset,
+        logging.CRITICAL: bold_red + fmt + reset,
+    }
+
+    def format(self, record: logging.LogRecord):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
+
 # For StreamHandlers / console
 class VerboseColorfulFormatter(CustomFormatter):
     def format(self, record):
@@ -32,7 +55,7 @@ class VerboseColorfulFormatter(CustomFormatter):
 
 # For Verbose FileHandlers / files
 class VerboseFormatter(CustomFormatter):
-    fmt = "%(asctime)s::%(name)s::%(levelname)s| %(message)s (%(filename)s:%(lineno)d)"
+    fmt = "%(asctime)s::%(name)s::%(levelname)s: %(message)s (%(filename)s:%(lineno)d)"
 
     def format(self, record):
         formatter = logging.Formatter(self.fmt)

{lmnr-0.4.47.dist-info → lmnr-0.4.49.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.47
+Version: 0.4.49
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -44,35 +44,35 @@ Requires-Dist: deprecated (>=1.0)
 Requires-Dist: opentelemetry-api (>=1.28.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.28.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.28.0)
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.33.12) ; extra == "all" or extra == "alephalpha"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.33.12) ; extra == "all" or extra == "anthropic"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.33.12) ; extra == "all" or extra == "bedrock"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.33.12) ; extra == "all" or extra == "chromadb"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.33.12) ; extra == "all" or extra == "cohere"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.33.12) ; extra == "all" or extra == "google-generativeai"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.33.12) ; extra == "all" or extra == "groq"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.33.12) ; extra == "all" or extra == "haystack"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.33.12) ; extra == "all" or extra == "lancedb"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.33.12) ; extra == "all" or extra == "langchain"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.33.12) ; extra == "all" or extra == "llamaindex"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.33.12) ; extra == "all" or extra == "marqo"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.33.12) ; extra == "all" or extra == "milvus"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.33.12) ; extra == "all" or extra == "mistralai"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.33.12) ; extra == "all" or extra == "ollama"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.34.0) ; extra == "all" or extra == "alephalpha"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.34.0) ; extra == "all" or extra == "anthropic"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.34.0) ; extra == "all" or extra == "bedrock"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.34.0) ; extra == "all" or extra == "chromadb"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.34.0) ; extra == "all" or extra == "cohere"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.34.0) ; extra == "all" or extra == "google-generativeai"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.34.0) ; extra == "all" or extra == "groq"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.34.0) ; extra == "all" or extra == "haystack"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.34.0) ; extra == "all" or extra == "lancedb"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.34.0) ; extra == "all" or extra == "langchain"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.34.0) ; extra == "all" or extra == "llamaindex"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.34.0) ; extra == "all" or extra == "marqo"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.34.0) ; extra == "all" or extra == "milvus"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.34.0) ; extra == "all" or extra == "mistralai"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.34.0) ; extra == "all" or extra == "ollama"
 Requires-Dist: opentelemetry-instrumentation-openai (>=0.33.12) ; extra == "all" or extra == "openai"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.33.12) ; extra == "all" or extra == "pinecone"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.33.12) ; extra == "all" or extra == "qdrant"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.33.12) ; extra == "all" or extra == "replicate"
-Requires-Dist: opentelemetry-instrumentation-requests (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.33.12) ; extra == "all" or extra == "sagemaker"
-Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-threading (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-together (>=0.33.12) ; extra == "all" or extra == "together"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.33.12) ; extra == "all" or extra == "transformers"
-Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.33.12) ; extra == "all" or extra == "vertexai"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.33.12) ; extra == "all" or extra == "watsonx"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.33.12) ; extra == "all" or extra == "weaviate"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.34.0) ; extra == "all" or extra == "pinecone"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.34.0) ; extra == "all" or extra == "qdrant"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.34.0) ; extra == "all" or extra == "replicate"
+Requires-Dist: opentelemetry-instrumentation-requests (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.34.0) ; extra == "all" or extra == "sagemaker"
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-threading (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-together (>=0.34.0) ; extra == "all" or extra == "together"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.34.0) ; extra == "all" or extra == "transformers"
+Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.34.0) ; extra == "all" or extra == "vertexai"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.34.0) ; extra == "all" or extra == "watsonx"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.34.0) ; extra == "all" or extra == "weaviate"
 Requires-Dist: opentelemetry-sdk (>=1.28.0)
 Requires-Dist: opentelemetry-semantic-conventions-ai (==0.4.2)
 Requires-Dist: pydantic (>=2.7)

{lmnr-0.4.47.dist-info → lmnr-0.4.49.dist-info}/RECORD

@@ -1,5 +1,5 @@
 lmnr/__init__.py,sha256=Bqxs-8Mh4h69pOHURgBCgo9EW1GwChebxP6wUX2-bsU,452
-lmnr/cli.py,sha256=W5zn5DBJiFaVARFy2D-8tJI8ZlGNzjJqk3qiXAG2woo,1456
+lmnr/cli.py,sha256=4J2RZQhHM3jJcjFvBC4PChQTS-ukxykVvI0X6lTkK-o,2918
 lmnr/openllmetry_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/openllmetry_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
 lmnr/openllmetry_sdk/__init__.py,sha256=vVSGTAwUnJvdulHtslkGAd8QCBuv78WUK3bgfBpH6Do,2390
@@ -21,13 +21,13 @@ lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/sdk/datasets.py,sha256=KNMp_v3z1ocIltIw7kTgj8o-l9R8N8Tgj0sw1ajQ9C8,1582
 lmnr/sdk/decorators.py,sha256=ja2EUWUWvFOp28ER0k78PRuxNahwCVyH0TdM3U-xY7U,1856
 lmnr/sdk/eval_control.py,sha256=G6Fg3Xx_KWv72iBaWlNMdyRTF2bZFQnwJ68sJNSpIcY,177
-lmnr/sdk/evaluations.py,sha256=gLImD_uB9uXgw07QiJ_OYRTFDGxiPtFCO1c8HyOq2s0,15935
-lmnr/sdk/laminar.py,sha256=4Saelm9m6pB9GWquCdHWY-1VhAB8Q2tWlq6hnmorzHU,31250
-lmnr/sdk/log.py,sha256=cZBeUoSK39LMEV-X4-eEhTWOciULRfHaKfRK8YqIM8I,1532
+lmnr/sdk/evaluations.py,sha256=lSXSvvYNB5JsSGli_wV-W34LVzJkQOiK8DnvHv4eU5A,16323
+lmnr/sdk/laminar.py,sha256=sUq04bSGfrNuSHOEMTbR01Me0ploX8SE5GIWKqUAVyY,31094
+lmnr/sdk/log.py,sha256=nt_YMmPw1IRbGy0b7q4rTtP4Yo3pQfNxqJPXK3nDSNQ,2213
 lmnr/sdk/types.py,sha256=FCNoFoa0ingOvpXGfbiETVsakYyq9Zpoc56MXJ1YDzQ,6390
 lmnr/sdk/utils.py,sha256=Uk8y15x-sd5tP2ERONahElLDJVEy_3dA_1_5g9A6auY,3358
-lmnr-0.4.47.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.47.dist-info/METADATA,sha256=dqgLT_aaBn35yjuzN0KKfI2BKVPLDIdfibN7PW5HWcw,12244
-lmnr-0.4.47.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lmnr-0.4.47.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.47.dist-info/RECORD,,
+lmnr-0.4.49.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.49.dist-info/METADATA,sha256=5VVoG_p8k1guIZWZXliEaa8qRCTQCU6pSLoiXuDo1TY,12196
+lmnr-0.4.49.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lmnr-0.4.49.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.49.dist-info/RECORD,,