lmnr 0.4.48__py3-none-any.whl → 0.4.50__py3-none-any.whl

lmnr/cli.py

@@ -1,40 +1,74 @@
 from argparse import ArgumentParser
 import asyncio
 import importlib.util
+import logging
 import os
+import re
 import sys
 
 from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
+from .sdk.log import ColorfulFormatter
+
+LOG = logging.getLogger(__name__)
+console_log_handler = logging.StreamHandler()
+console_log_handler.setFormatter(ColorfulFormatter())
+LOG.addHandler(console_log_handler)
+
+
+EVAL_DIR = "evals"
 
 
 async def run_evaluation(args):
     sys.path.append(os.getcwd())
 
-    prep_token = PREPARE_ONLY.set(True)
-    try:
-        file = os.path.abspath(args.file)
-        name = "user_module"
+    if args.file is None:
+        files = [
+            os.path.join(EVAL_DIR, f)
+            for f in os.listdir(EVAL_DIR)
+            if re.match(r".*_eval\.py$", f) or re.match(r"eval_.*\.py$", f)
+        ]
+        if len(files) == 0:
+            LOG.error("No evaluation files found in evals directory")
+            return
+        files.sort()
+        LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
+
+    else:
+        files = [args.file]
 
-        spec = importlib.util.spec_from_file_location(name, file)
-        if spec is None or spec.loader is None:
-            raise ImportError(f"Could not load module specification from {file}")
-        mod = importlib.util.module_from_spec(spec)
-        sys.modules[name] = mod
+    for file in files:
+        prep_token = PREPARE_ONLY.set(True)
+        LOG.info(f"Running evaluation from {file}")
+        try:
+            file = os.path.abspath(file)
+            name = "user_module" + file
 
-        spec.loader.exec_module(mod)
-        evaluation = EVALUATION_INSTANCE.get()
-        if evaluation is None:
-            raise RuntimeError("Evaluation instance not found")
+            spec = importlib.util.spec_from_file_location(name, file)
+            if spec is None or spec.loader is None:
+                LOG.error(f"Could not load module specification from {file}")
+                if args.fail_on_error:
+                    return
+                continue
+            mod = importlib.util.module_from_spec(spec)
+            sys.modules[name] = mod
 
-        await evaluation.run()
-    finally:
-        PREPARE_ONLY.reset(prep_token)
+            spec.loader.exec_module(mod)
+            evaluation = EVALUATION_INSTANCE.get()
+            if evaluation is None:
+                LOG.warning("Evaluation instance not found")
+                if args.fail_on_error:
+                    return
+                continue
+
+            await evaluation.run()
+        finally:
+            PREPARE_ONLY.reset(prep_token)
 
 
 def cli():
     parser = ArgumentParser(
         prog="lmnr",
-        description="CLI for Laminar",
+        description="CLI for Laminar. Call `lmnr [subcommand] --help` for more information on each subcommand.",
     )
 
     subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
@@ -44,7 +78,21 @@ def cli():
         description="Run an evaluation",
         help="Run an evaluation",
     )
-    parser_eval.add_argument("file", help="A file containing the evaluation to run")
+    parser_eval.add_argument(
+        "file",
+        nargs="?",
+        help="A file containing the evaluation to run."
+        + "If no file name is provided, all evaluation files in the `evals` directory are run as long"
+        + "as they match *_eval.py or eval_*.py",
+        default=None,
+    )
+
+    parser_eval.add_argument(
+        "--fail-on-error",
+        action="store_true",
+        default=False,
+        help="Fail on error",
+    )
 
     parsed = parser.parse_args()
     if parsed.subcommand == "eval":

lmnr/openllmetry_sdk/tracing/tracing.py

@@ -509,7 +509,7 @@ def init_chroma_instrumentor():
 
 def init_google_generativeai_instrumentor():
     try:
-        if is_package_installed("google.generativeai") and is_package_installed(
+        if is_package_installed("google-generativeai") and is_package_installed(
             "opentelemetry-instrumentation-google-generativeai"
         ):
             from opentelemetry.instrumentation.google_generativeai import (

lmnr/sdk/evaluations.py

@@ -29,8 +29,17 @@ from .utils import is_async
 DEFAULT_BATCH_SIZE = 5
 
 
-def get_evaluation_url(project_id: str, evaluation_id: str):
-    return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
+def get_evaluation_url(
+    project_id: str, evaluation_id: str, base_url: str = "https://www.lmnr.ai"
+):
+    url = base_url
+    if url.endswith("/"):
+        url = url[:-1]
+    if url.endswith("localhost") or url.endswith("127.0.0.1"):
+        # We best effort assume that the frontend is running on port 3000
+        # TODO: expose the frontend port?
+        url = url + ":3000"
+    return f"{url}/project/{project_id}/evaluations/{evaluation_id}"
 
 
 def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Numeric]:
@@ -49,8 +58,8 @@ def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Nu
 
 
 class EvaluationReporter:
-    def __init__(self):
-        pass
+    def __init__(self, base_url: str = "https://www.lmnr.ai"):
+        self.base_url = base_url
 
     def start(self, length: int):
         self.cli_progress = tqdm(
@@ -71,7 +80,7 @@ class EvaluationReporter:
     ):
         self.cli_progress.close()
         print(
-            f"\nCheck the results at {get_evaluation_url(project_id, evaluation_id)}\n"
+            f"\nCheck the results at {get_evaluation_url(project_id, evaluation_id, self.base_url)}\n"
         )
         print("Average scores:")
         for name, score in average_scores.items():
@@ -160,7 +169,7 @@ class Evaluation:
                 )
 
         self.is_finished = False
-        self.reporter = EvaluationReporter()
+        self.reporter = EvaluationReporter(base_url)
         if isinstance(data, list):
             self.data = [
                 (Datapoint.model_validate(point) if isinstance(point, dict) else point)

lmnr/sdk/laminar.py

@@ -136,15 +136,7 @@ class Laminar:
         cls.__initialized = True
         cls._initialize_logger()
 
-        if _processor is not None:
-            cls.__logger.warning(
-                "Using a custom span processor. This feature is added for tests only. "
-                "Any use of this feature outside of tests is not supported and "
-                "advised against."
-            )
-
         Traceloop.init(
-            processor=_processor,
             exporter=OTLPSpanExporter(
                 endpoint=cls.__base_grpc_url,
                 headers={"authorization": f"Bearer {cls.__project_api_key}"},
@@ -407,7 +399,7 @@ class Laminar:
                     )
                 yield span
 
-            # # TODO: Figure out if this is necessary
+            # TODO: Figure out if this is necessary
             try:
                 detach(ctx_token)
             except Exception:

lmnr/sdk/log.py

@@ -24,6 +24,29 @@ class CustomFormatter(logging.Formatter):
         return formatter.format(record)
 
 
+class ColorfulFormatter(logging.Formatter):
+    grey = "\x1b[38;20m"
+    green = "\x1b[32;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    fmt = "Laminar %(levelname)s: %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: grey + fmt + reset,
+        logging.INFO: green + fmt + reset,
+        logging.WARNING: yellow + fmt + reset,
+        logging.ERROR: red + fmt + reset,
+        logging.CRITICAL: bold_red + fmt + reset,
+    }
+
+    def format(self, record: logging.LogRecord):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
+
 # For StreamHandlers / console
 class VerboseColorfulFormatter(CustomFormatter):
     def format(self, record):
@@ -32,7 +55,7 @@ class VerboseColorfulFormatter(CustomFormatter):
 
 # For Verbose FileHandlers / files
 class VerboseFormatter(CustomFormatter):
-    fmt = "%(asctime)s::%(name)s::%(levelname)s| %(message)s (%(filename)s:%(lineno)d)"
+    fmt = "%(asctime)s::%(name)s::%(levelname)s: %(message)s (%(filename)s:%(lineno)d)"
 
     def format(self, record):
         formatter = logging.Formatter(self.fmt)

{lmnr-0.4.48.dist-info → lmnr-0.4.50.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.48
+Version: 0.4.50
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -44,35 +44,35 @@ Requires-Dist: deprecated (>=1.0)
 Requires-Dist: opentelemetry-api (>=1.28.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.28.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.28.0)
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.33.12) ; extra == "all" or extra == "alephalpha"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.33.12) ; extra == "all" or extra == "anthropic"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.33.12) ; extra == "all" or extra == "bedrock"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.33.12) ; extra == "all" or extra == "chromadb"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.33.12) ; extra == "all" or extra == "cohere"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.33.12) ; extra == "all" or extra == "google-generativeai"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.33.12) ; extra == "all" or extra == "groq"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.33.12) ; extra == "all" or extra == "haystack"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.33.12) ; extra == "all" or extra == "lancedb"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.33.12) ; extra == "all" or extra == "langchain"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.33.12) ; extra == "all" or extra == "llamaindex"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.33.12) ; extra == "all" or extra == "marqo"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.33.12) ; extra == "all" or extra == "milvus"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.33.12) ; extra == "all" or extra == "mistralai"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.33.12) ; extra == "all" or extra == "ollama"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.34.0) ; extra == "all" or extra == "alephalpha"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.34.0) ; extra == "all" or extra == "anthropic"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.34.0) ; extra == "all" or extra == "bedrock"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.34.0) ; extra == "all" or extra == "chromadb"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.34.0) ; extra == "all" or extra == "cohere"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.34.0) ; extra == "all" or extra == "google-generativeai"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.34.0) ; extra == "all" or extra == "groq"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.34.0) ; extra == "all" or extra == "haystack"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.34.0) ; extra == "all" or extra == "lancedb"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.34.0) ; extra == "all" or extra == "langchain"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.34.0) ; extra == "all" or extra == "llamaindex"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.34.0) ; extra == "all" or extra == "marqo"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.34.0) ; extra == "all" or extra == "milvus"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.34.0) ; extra == "all" or extra == "mistralai"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.34.0) ; extra == "all" or extra == "ollama"
 Requires-Dist: opentelemetry-instrumentation-openai (>=0.33.12) ; extra == "all" or extra == "openai"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.33.12) ; extra == "all" or extra == "pinecone"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.33.12) ; extra == "all" or extra == "qdrant"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.33.12) ; extra == "all" or extra == "replicate"
-Requires-Dist: opentelemetry-instrumentation-requests (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.33.12) ; extra == "all" or extra == "sagemaker"
-Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-threading (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-together (>=0.33.12) ; extra == "all" or extra == "together"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.33.12) ; extra == "all" or extra == "transformers"
-Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.33.12) ; extra == "all" or extra == "vertexai"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.33.12) ; extra == "all" or extra == "watsonx"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.33.12) ; extra == "all" or extra == "weaviate"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.34.0) ; extra == "all" or extra == "pinecone"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.34.0) ; extra == "all" or extra == "qdrant"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.34.0) ; extra == "all" or extra == "replicate"
+Requires-Dist: opentelemetry-instrumentation-requests (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.34.0) ; extra == "all" or extra == "sagemaker"
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-threading (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-together (>=0.34.0) ; extra == "all" or extra == "together"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.34.0) ; extra == "all" or extra == "transformers"
+Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.34.0) ; extra == "all" or extra == "vertexai"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.34.0) ; extra == "all" or extra == "watsonx"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.34.0) ; extra == "all" or extra == "weaviate"
 Requires-Dist: opentelemetry-sdk (>=1.28.0)
 Requires-Dist: opentelemetry-semantic-conventions-ai (==0.4.2)
 Requires-Dist: pydantic (>=2.7)

{lmnr-0.4.48.dist-info → lmnr-0.4.50.dist-info}/RECORD

@@ -1,5 +1,5 @@
 lmnr/__init__.py,sha256=Bqxs-8Mh4h69pOHURgBCgo9EW1GwChebxP6wUX2-bsU,452
-lmnr/cli.py,sha256=W5zn5DBJiFaVARFy2D-8tJI8ZlGNzjJqk3qiXAG2woo,1456
+lmnr/cli.py,sha256=4J2RZQhHM3jJcjFvBC4PChQTS-ukxykVvI0X6lTkK-o,2918
 lmnr/openllmetry_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
 lmnr/openllmetry_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
 lmnr/openllmetry_sdk/__init__.py,sha256=vVSGTAwUnJvdulHtslkGAd8QCBuv78WUK3bgfBpH6Do,2390
@@ -11,7 +11,7 @@ lmnr/openllmetry_sdk/tracing/__init__.py,sha256=xT73L1t2si2CM6QmMiTZ7zn-dKKYBLNr
 lmnr/openllmetry_sdk/tracing/attributes.py,sha256=B_4KVYWAUu-6DQmsm2eCJQcTxm8pG1EByCBK3uOPkuI,1293
 lmnr/openllmetry_sdk/tracing/content_allow_list.py,sha256=3feztm6PBWNelc8pAZUcQyEGyeSpNiVKjOaDk65l2ps,846
 lmnr/openllmetry_sdk/tracing/context_manager.py,sha256=rdSus-p-TaevQ8hIAhfbnZr5dTqRvACDkzXGDpflncY,306
-lmnr/openllmetry_sdk/tracing/tracing.py,sha256=pa8ZPhH4NSwP1QnvD5qNaOZ775H5Q4KVa9OwSki9KUE,32138
+lmnr/openllmetry_sdk/tracing/tracing.py,sha256=X87Dw-WhtGP-THeiTRzZ1FBosRdY2FR1xEcE09PgkRI,32138
 lmnr/openllmetry_sdk/utils/__init__.py,sha256=pNhf0G3vTd5ccoc03i1MXDbricSaiqCbi1DLWhSekK8,604
 lmnr/openllmetry_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQvzJk_fZH0OOsRAM1iZQXsR8,2112
 lmnr/openllmetry_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
@@ -21,13 +21,13 @@ lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 lmnr/sdk/datasets.py,sha256=KNMp_v3z1ocIltIw7kTgj8o-l9R8N8Tgj0sw1ajQ9C8,1582
 lmnr/sdk/decorators.py,sha256=ja2EUWUWvFOp28ER0k78PRuxNahwCVyH0TdM3U-xY7U,1856
 lmnr/sdk/eval_control.py,sha256=G6Fg3Xx_KWv72iBaWlNMdyRTF2bZFQnwJ68sJNSpIcY,177
-lmnr/sdk/evaluations.py,sha256=gLImD_uB9uXgw07QiJ_OYRTFDGxiPtFCO1c8HyOq2s0,15935
-lmnr/sdk/laminar.py,sha256=pgvkCswo9csOIWD3DWr6EW2UqMbUWc-AomVVOGD2U2E,31418
-lmnr/sdk/log.py,sha256=cZBeUoSK39LMEV-X4-eEhTWOciULRfHaKfRK8YqIM8I,1532
+lmnr/sdk/evaluations.py,sha256=lSXSvvYNB5JsSGli_wV-W34LVzJkQOiK8DnvHv4eU5A,16323
+lmnr/sdk/laminar.py,sha256=sUq04bSGfrNuSHOEMTbR01Me0ploX8SE5GIWKqUAVyY,31094
+lmnr/sdk/log.py,sha256=nt_YMmPw1IRbGy0b7q4rTtP4Yo3pQfNxqJPXK3nDSNQ,2213
 lmnr/sdk/types.py,sha256=FCNoFoa0ingOvpXGfbiETVsakYyq9Zpoc56MXJ1YDzQ,6390
 lmnr/sdk/utils.py,sha256=Uk8y15x-sd5tP2ERONahElLDJVEy_3dA_1_5g9A6auY,3358
-lmnr-0.4.48.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
-lmnr-0.4.48.dist-info/METADATA,sha256=KCZg8NS72mThs3tAJ4SL9OwDSTcvsosx5JxzpDTtVTw,12244
-lmnr-0.4.48.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-lmnr-0.4.48.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
-lmnr-0.4.48.dist-info/RECORD,,
+lmnr-0.4.50.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.50.dist-info/METADATA,sha256=Qpu8olLKxcpmaPPf1aczP16sRM_4KyHwkueRLp56EKM,12196
+lmnr-0.4.50.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+lmnr-0.4.50.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
+lmnr-0.4.50.dist-info/RECORD,,