lmnr 0.4.48__tar.gz → 0.4.49__tar.gz

{lmnr-0.4.48 → lmnr-0.4.49}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.48
+Version: 0.4.49
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -44,35 +44,35 @@ Requires-Dist: deprecated (>=1.0)
 Requires-Dist: opentelemetry-api (>=1.28.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.28.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.28.0)
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.33.12) ; extra == "all" or extra == "alephalpha"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.33.12) ; extra == "all" or extra == "anthropic"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.33.12) ; extra == "all" or extra == "bedrock"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.33.12) ; extra == "all" or extra == "chromadb"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.33.12) ; extra == "all" or extra == "cohere"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.33.12) ; extra == "all" or extra == "google-generativeai"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.33.12) ; extra == "all" or extra == "groq"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.33.12) ; extra == "all" or extra == "haystack"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.33.12) ; extra == "all" or extra == "lancedb"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.33.12) ; extra == "all" or extra == "langchain"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.33.12) ; extra == "all" or extra == "llamaindex"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.33.12) ; extra == "all" or extra == "marqo"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.33.12) ; extra == "all" or extra == "milvus"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.33.12) ; extra == "all" or extra == "mistralai"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.33.12) ; extra == "all" or extra == "ollama"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.34.0) ; extra == "all" or extra == "alephalpha"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.34.0) ; extra == "all" or extra == "anthropic"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.34.0) ; extra == "all" or extra == "bedrock"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.34.0) ; extra == "all" or extra == "chromadb"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.34.0) ; extra == "all" or extra == "cohere"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.34.0) ; extra == "all" or extra == "google-generativeai"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.34.0) ; extra == "all" or extra == "groq"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.34.0) ; extra == "all" or extra == "haystack"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.34.0) ; extra == "all" or extra == "lancedb"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.34.0) ; extra == "all" or extra == "langchain"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.34.0) ; extra == "all" or extra == "llamaindex"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.34.0) ; extra == "all" or extra == "marqo"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.34.0) ; extra == "all" or extra == "milvus"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.34.0) ; extra == "all" or extra == "mistralai"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.34.0) ; extra == "all" or extra == "ollama"
 Requires-Dist: opentelemetry-instrumentation-openai (>=0.33.12) ; extra == "all" or extra == "openai"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.33.12) ; extra == "all" or extra == "pinecone"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.33.12) ; extra == "all" or extra == "qdrant"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.33.12) ; extra == "all" or extra == "replicate"
-Requires-Dist: opentelemetry-instrumentation-requests (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.33.12) ; extra == "all" or extra == "sagemaker"
-Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-threading (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-together (>=0.33.12) ; extra == "all" or extra == "together"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.33.12) ; extra == "all" or extra == "transformers"
-Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.49b0,<0.50)
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.33.12) ; extra == "all" or extra == "vertexai"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.33.12) ; extra == "all" or extra == "watsonx"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.33.12) ; extra == "all" or extra == "weaviate"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.34.0) ; extra == "all" or extra == "pinecone"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.34.0) ; extra == "all" or extra == "qdrant"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.34.0) ; extra == "all" or extra == "replicate"
+Requires-Dist: opentelemetry-instrumentation-requests (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.34.0) ; extra == "all" or extra == "sagemaker"
+Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-threading (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-together (>=0.34.0) ; extra == "all" or extra == "together"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.34.0) ; extra == "all" or extra == "transformers"
+Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.50b0)
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.34.0) ; extra == "all" or extra == "vertexai"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.34.0) ; extra == "all" or extra == "watsonx"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.34.0) ; extra == "all" or extra == "weaviate"
 Requires-Dist: opentelemetry-sdk (>=1.28.0)
 Requires-Dist: opentelemetry-semantic-conventions-ai (==0.4.2)
 Requires-Dist: pydantic (>=2.7)

{lmnr-0.4.48 → lmnr-0.4.49}/pyproject.toml

@@ -6,7 +6,7 @@
 
 [project]
 name = "lmnr"
-version = "0.4.48"
+version = "0.4.49"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -17,7 +17,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.48"
+version = "0.4.49"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"
@@ -32,41 +32,41 @@ opentelemetry-api = ">=1.28.0"
 opentelemetry-sdk = ">=1.28.0"
 opentelemetry-exporter-otlp-proto-http = ">=1.28.0"
 opentelemetry-exporter-otlp-proto-grpc = ">=1.28.0"
-opentelemetry-instrumentation-requests = "^0.49b0"
-opentelemetry-instrumentation-sqlalchemy = "^0.49b0"
-opentelemetry-instrumentation-urllib3 = "^0.49b0"
-opentelemetry-instrumentation-threading = "^0.49b0"
+opentelemetry-instrumentation-requests = ">=0.50b0"
+opentelemetry-instrumentation-sqlalchemy = ">=0.50b0"
+opentelemetry-instrumentation-urllib3 = ">=0.50b0"
+opentelemetry-instrumentation-threading = ">=0.50b0"
 opentelemetry-semantic-conventions-ai = "0.4.2"
 deprecated = ">=1.0"
 tqdm = ">=4.0"
 argparse = ">=1.0"
 aiohttp = ">=3.0"
 tenacity = ">=8.0"
-opentelemetry-instrumentation-alephalpha = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-anthropic = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-bedrock = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-chromadb = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-cohere = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-google-generativeai = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-groq = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-haystack = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-lancedb = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-langchain = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-llamaindex = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-marqo = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-milvus = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-mistralai = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-ollama = {version = ">=0.33.12", optional = true}
+opentelemetry-instrumentation-alephalpha = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-anthropic = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-bedrock = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-chromadb = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-cohere = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-google-generativeai = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-groq = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-haystack = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-lancedb = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-langchain = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-llamaindex = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-marqo = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-milvus = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-mistralai = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-ollama = {version = ">=0.34.0", optional = true}
 opentelemetry-instrumentation-openai = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-pinecone = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-qdrant = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-replicate = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-sagemaker = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-together = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-transformers = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-vertexai = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-watsonx = {version = ">=0.33.12", optional = true}
-opentelemetry-instrumentation-weaviate = {version = ">=0.33.12", optional = true}
+opentelemetry-instrumentation-pinecone = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-qdrant = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-replicate = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-sagemaker = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-together = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-transformers = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-vertexai = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-watsonx = {version = ">=0.34.0", optional = true}
+opentelemetry-instrumentation-weaviate = {version = ">=0.34.0", optional = true}
 
 [tool.poetry.extras]
 # List of all possible extras. You can specify one or more of these extras

lmnr-0.4.49/src/lmnr/cli.py

@@ -0,0 +1,101 @@
+from argparse import ArgumentParser
+import asyncio
+import importlib.util
+import logging
+import os
+import re
+import sys
+
+from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
+from .sdk.log import ColorfulFormatter
+
+LOG = logging.getLogger(__name__)
+console_log_handler = logging.StreamHandler()
+console_log_handler.setFormatter(ColorfulFormatter())
+LOG.addHandler(console_log_handler)
+
+
+EVAL_DIR = "evals"
+
+
+async def run_evaluation(args):
+    sys.path.append(os.getcwd())
+
+    if args.file is None:
+        files = [
+            os.path.join(EVAL_DIR, f)
+            for f in os.listdir(EVAL_DIR)
+            if re.match(r".*_eval\.py$", f) or re.match(r"eval_.*\.py$", f)
+        ]
+        if len(files) == 0:
+            LOG.error("No evaluation files found in evals directory")
+            return
+        files.sort()
+        LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
+
+    else:
+        files = [args.file]
+
+    for file in files:
+        prep_token = PREPARE_ONLY.set(True)
+        LOG.info(f"Running evaluation from {file}")
+        try:
+            file = os.path.abspath(file)
+            name = "user_module" + file
+
+            spec = importlib.util.spec_from_file_location(name, file)
+            if spec is None or spec.loader is None:
+                LOG.error(f"Could not load module specification from {file}")
+                if args.fail_on_error:
+                    return
+                continue
+            mod = importlib.util.module_from_spec(spec)
+            sys.modules[name] = mod
+
+            spec.loader.exec_module(mod)
+            evaluation = EVALUATION_INSTANCE.get()
+            if evaluation is None:
+                LOG.warning("Evaluation instance not found")
+                if args.fail_on_error:
+                    return
+                continue
+
+            await evaluation.run()
+        finally:
+            PREPARE_ONLY.reset(prep_token)
+
+
+def cli():
+    parser = ArgumentParser(
+        prog="lmnr",
+        description="CLI for Laminar. Call `lmnr [subcommand] --help` for more information on each subcommand.",
+    )
+
+    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
+
+    parser_eval = subparsers.add_parser(
+        "eval",
+        description="Run an evaluation",
+        help="Run an evaluation",
+    )
+    parser_eval.add_argument(
+        "file",
+        nargs="?",
+        help="A file containing the evaluation to run."
+        + "If no file name is provided, all evaluation files in the `evals` directory are run as long"
+        + "as they match *_eval.py or eval_*.py",
+        default=None,
+    )
+
+    parser_eval.add_argument(
+        "--fail-on-error",
+        action="store_true",
+        default=False,
+        help="Fail on error",
+    )
+
+    parsed = parser.parse_args()
+    if parsed.subcommand == "eval":
+        asyncio.run(run_evaluation(parsed))
+    else:
+        parser.print_help()

{lmnr-0.4.48 → lmnr-0.4.49}/src/lmnr/sdk/evaluations.py

@@ -29,8 +29,17 @@ from .utils import is_async
 DEFAULT_BATCH_SIZE = 5
 
 
-def get_evaluation_url(project_id: str, evaluation_id: str):
-    return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
+def get_evaluation_url(
+    project_id: str, evaluation_id: str, base_url: str = "https://www.lmnr.ai"
+):
+    url = base_url
+    if url.endswith("/"):
+        url = url[:-1]
+    if url.endswith("localhost") or url.endswith("127.0.0.1"):
+        # We best effort assume that the frontend is running on port 3000
+        # TODO: expose the frontend port?
+        url = url + ":3000"
+    return f"{url}/project/{project_id}/evaluations/{evaluation_id}"
 
 
 def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Numeric]:
@@ -49,8 +58,8 @@ def get_average_scores(results: list[EvaluationResultDatapoint]) -> dict[str, Nu
 
 
 class EvaluationReporter:
-    def __init__(self):
-        pass
+    def __init__(self, base_url: str = "https://www.lmnr.ai"):
+        self.base_url = base_url
 
     def start(self, length: int):
         self.cli_progress = tqdm(
@@ -71,7 +80,7 @@ class EvaluationReporter:
     ):
         self.cli_progress.close()
         print(
-            f"\nCheck the results at {get_evaluation_url(project_id, evaluation_id)}\n"
+            f"\nCheck the results at {get_evaluation_url(project_id, evaluation_id, self.base_url)}\n"
         )
         print("Average scores:")
         for name, score in average_scores.items():
@@ -160,7 +169,7 @@ class Evaluation:
                 )
 
         self.is_finished = False
-        self.reporter = EvaluationReporter()
+        self.reporter = EvaluationReporter(base_url)
         if isinstance(data, list):
             self.data = [
                 (Datapoint.model_validate(point) if isinstance(point, dict) else point)

{lmnr-0.4.48 → lmnr-0.4.49}/src/lmnr/sdk/laminar.py

@@ -136,15 +136,7 @@ class Laminar:
         cls.__initialized = True
         cls._initialize_logger()
 
-        if _processor is not None:
-            cls.__logger.warning(
-                "Using a custom span processor. This feature is added for tests only. "
-                "Any use of this feature outside of tests is not supported and "
-                "advised against."
-            )
-
         Traceloop.init(
-            processor=_processor,
             exporter=OTLPSpanExporter(
                 endpoint=cls.__base_grpc_url,
                 headers={"authorization": f"Bearer {cls.__project_api_key}"},
@@ -407,7 +399,7 @@ class Laminar:
                     )
                 yield span
 
-            # # TODO: Figure out if this is necessary
+            # TODO: Figure out if this is necessary
             try:
                 detach(ctx_token)
             except Exception:

{lmnr-0.4.48 → lmnr-0.4.49}/src/lmnr/sdk/log.py

@@ -24,6 +24,29 @@ class CustomFormatter(logging.Formatter):
         return formatter.format(record)
 
 
+class ColorfulFormatter(logging.Formatter):
+    grey = "\x1b[38;20m"
+    green = "\x1b[32;20m"
+    yellow = "\x1b[33;20m"
+    red = "\x1b[31;20m"
+    bold_red = "\x1b[31;1m"
+    reset = "\x1b[0m"
+    fmt = "Laminar %(levelname)s: %(message)s"
+
+    FORMATS = {
+        logging.DEBUG: grey + fmt + reset,
+        logging.INFO: green + fmt + reset,
+        logging.WARNING: yellow + fmt + reset,
+        logging.ERROR: red + fmt + reset,
+        logging.CRITICAL: bold_red + fmt + reset,
+    }
+
+    def format(self, record: logging.LogRecord):
+        log_fmt = self.FORMATS.get(record.levelno)
+        formatter = logging.Formatter(log_fmt)
+        return formatter.format(record)
+
+
 # For StreamHandlers / console
 class VerboseColorfulFormatter(CustomFormatter):
     def format(self, record):
@@ -32,7 +55,7 @@ class VerboseColorfulFormatter(CustomFormatter):
 
 # For Verbose FileHandlers / files
 class VerboseFormatter(CustomFormatter):
-    fmt = "%(asctime)s::%(name)s::%(levelname)s| %(message)s (%(filename)s:%(lineno)d)"
+    fmt = "%(asctime)s::%(name)s::%(levelname)s: %(message)s (%(filename)s:%(lineno)d)"
 
     def format(self, record):
         formatter = logging.Formatter(self.fmt)

lmnr-0.4.48/src/lmnr/cli.py

@@ -1,53 +0,0 @@
-from argparse import ArgumentParser
-import asyncio
-import importlib.util
-import os
-import sys
-
-from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
-
-
-async def run_evaluation(args):
-    sys.path.append(os.getcwd())
-
-    prep_token = PREPARE_ONLY.set(True)
-    try:
-        file = os.path.abspath(args.file)
-        name = "user_module"
-
-        spec = importlib.util.spec_from_file_location(name, file)
-        if spec is None or spec.loader is None:
-            raise ImportError(f"Could not load module specification from {file}")
-        mod = importlib.util.module_from_spec(spec)
-        sys.modules[name] = mod
-
-        spec.loader.exec_module(mod)
-        evaluation = EVALUATION_INSTANCE.get()
-        if evaluation is None:
-            raise RuntimeError("Evaluation instance not found")
-
-        await evaluation.run()
-    finally:
-        PREPARE_ONLY.reset(prep_token)
-
-
-def cli():
-    parser = ArgumentParser(
-        prog="lmnr",
-        description="CLI for Laminar",
-    )
-
-    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
-
-    parser_eval = subparsers.add_parser(
-        "eval",
-        description="Run an evaluation",
-        help="Run an evaluation",
-    )
-    parser_eval.add_argument("file", help="A file containing the evaluation to run")
-
-    parsed = parser.parse_args()
-    if parsed.subcommand == "eval":
-        asyncio.run(run_evaluation(parsed))
-    else:
-        parser.print_help()