lmnr 0.6.8__tar.gz → 0.6.10__tar.gz

{lmnr-0.6.8 → lmnr-0.6.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: lmnr
-Version: 0.6.8
+Version: 0.6.10
 Summary: Python SDK for Laminar
 License: Apache-2.0
 Author: lmnr.ai
@@ -46,61 +46,61 @@ Requires-Dist: httpx (>=0.25.0)
 Requires-Dist: opentelemetry-api (>=1.33.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.33.0)
 Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.33.0)
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.5) ; extra == "alephalpha"
-Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.5) ; extra == "anthropic"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.5) ; extra == "bedrock"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.5) ; extra == "chromadb"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.5) ; extra == "cohere"
-Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.5) ; extra == "crewai"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.5) ; extra == "google-generativeai"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.5) ; extra == "groq"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.5) ; extra == "haystack"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.5) ; extra == "lancedb"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.5) ; extra == "langchain"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.5) ; extra == "llamaindex"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.5) ; extra == "marqo"
-Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.5) ; extra == "mcp"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.5) ; extra == "milvus"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.5) ; extra == "mistralai"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.5) ; extra == "ollama"
-Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.5) ; extra == "openai"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.5) ; extra == "pinecone"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.5) ; extra == "qdrant"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.5) ; extra == "replicate"
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.5) ; extra == "sagemaker"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.8) ; extra == "alephalpha"
+Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.40.8) ; extra == "anthropic"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.40.8) ; extra == "bedrock"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.40.8) ; extra == "chromadb"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-cohere (>=0.40.8) ; extra == "cohere"
+Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-crewai (>=0.40.8) ; extra == "crewai"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.40.8) ; extra == "google-generativeai"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-groq (>=0.40.8) ; extra == "groq"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-haystack (>=0.40.8) ; extra == "haystack"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.40.8) ; extra == "lancedb"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-langchain (>=0.40.8) ; extra == "langchain"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.40.8) ; extra == "llamaindex"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-marqo (>=0.40.8) ; extra == "marqo"
+Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-mcp (>=0.40.8) ; extra == "mcp"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-milvus (>=0.40.8) ; extra == "milvus"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.40.8) ; extra == "mistralai"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-ollama (>=0.40.8) ; extra == "ollama"
+Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-openai (>=0.40.8) ; extra == "openai"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.40.8) ; extra == "pinecone"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.40.8) ; extra == "qdrant"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-replicate (>=0.40.8) ; extra == "replicate"
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.40.8) ; extra == "sagemaker"
 Requires-Dist: opentelemetry-instrumentation-threading (>=0.54b0)
-Requires-Dist: opentelemetry-instrumentation-together (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-together (>=0.40.5) ; extra == "together"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.5) ; extra == "transformers"
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.5) ; extra == "vertexai"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.5) ; extra == "watsonx"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.5) ; extra == "all"
-Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.5) ; extra == "weaviate"
+Requires-Dist: opentelemetry-instrumentation-together (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-together (>=0.40.8) ; extra == "together"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-transformers (>=0.40.8) ; extra == "transformers"
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.40.8) ; extra == "vertexai"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.40.8) ; extra == "watsonx"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.8) ; extra == "all"
+Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.40.8) ; extra == "weaviate"
 Requires-Dist: opentelemetry-sdk (>=1.33.0)
 Requires-Dist: opentelemetry-semantic-conventions (>=0.54b0)
 Requires-Dist: opentelemetry-semantic-conventions-ai (>=0.4.8)

lmnr-0.6.10/pyproject.toml

@@ -0,0 +1,142 @@
+# Laminar Python
+
+# If you are looking for information about possible extras installations,
+# i.e. what you can pass into `pip install 'lmnr[extra1,extra2]'`, please see the
+# `[project.optional-dependencies]` section below.
+
+[project]
+name = "lmnr"
+version = "0.6.10"
+description = "Python SDK for Laminar"
+authors = [
+  { name = "lmnr.ai", email = "founders@lmnr.ai" }
+]
+readme = "README.md"
+requires-python = ">=3.10,<4"
+license = "Apache-2.0"
+dependencies = [
+  "pydantic (>=2.0.3,<3.0.0)",
+  "python-dotenv (>=1.0)",
+  "opentelemetry-api (>=1.33.0)",
+  "opentelemetry-sdk (>=1.33.0)",
+  "opentelemetry-exporter-otlp-proto-http (>=1.33.0)",
+  "opentelemetry-exporter-otlp-proto-grpc (>=1.33.0)",
+  "opentelemetry-semantic-conventions (>=0.54b0)",
+  "opentelemetry-semantic-conventions-ai (>=0.4.8)",
+  "tqdm (>=4.0)",
+  "argparse (>=1.0)",
+  "tenacity (>=8.0)",
+  # Since 1.68.0, grpcio writes a warning message
+  # that looks scary, but is harmless.
+  # WARNING: All log messages before absl::InitializeLog() is called are written to STDERR
+  # E0000 00:00:1737439981.199902 9456033 init.cc:229] grpc_wait_for_shutdown_with_timeout() timed out.
+  #
+  # Remove this comment when we make sure that grpcio has resolved this.
+  # Related issue:
+  # https://discuss.ai.google.dev/t/warning-all-log-messages-before-absl-initializelog-is-called-are-written-to-stderr-e0000-001731955515-629532-17124-init-cc-229-grpc-wait-for-shutdown-with-timeout-timed-out/50020
+  # https://github.com/grpc/grpc/issues/38490
+  "grpcio>=1",
+  "httpx>=0.25.0",
+  "opentelemetry-instrumentation-threading>=0.54b0",
+]
+
+[project.scripts]
+lmnr = "lmnr.cli:cli"
+
+[project.optional-dependencies]
+# List of all possible extras. You can specify one or more of these extras
+# when installing the package, using any of the following examples:
+# `pip install 'lmnr[anthropic,openai]'`
+# `uv pip install 'lmnr[anthropic,openai]'`
+# `uv add lmnr --extra anthropic --extra openai`
+# `poetry add 'lmnr[anthropic,openai]'`
+
+alephalpha=["opentelemetry-instrumentation-alephalpha>=0.40.8"]
+anthropic=["opentelemetry-instrumentation-anthropic>=0.40.8"]
+bedrock=["opentelemetry-instrumentation-bedrock>=0.40.8"]
+chromadb=["opentelemetry-instrumentation-chromadb>=0.40.8"]
+cohere=["opentelemetry-instrumentation-cohere>=0.40.8"]
+crewai=["opentelemetry-instrumentation-crewai>=0.40.8"]
+google-generativeai=["opentelemetry-instrumentation-google-generativeai>=0.40.8"]
+groq=["opentelemetry-instrumentation-groq>=0.40.8"]
+haystack=["opentelemetry-instrumentation-haystack>=0.40.8"]
+lancedb=["opentelemetry-instrumentation-lancedb>=0.40.8"]
+langchain=["opentelemetry-instrumentation-langchain>=0.40.8"]
+llamaindex=["opentelemetry-instrumentation-llamaindex>=0.40.8"]
+marqo=["opentelemetry-instrumentation-marqo>=0.40.8"]
+mcp=["opentelemetry-instrumentation-mcp>=0.40.8"]
+milvus=["opentelemetry-instrumentation-milvus>=0.40.8"]
+mistralai=["opentelemetry-instrumentation-mistralai>=0.40.8"]
+ollama=["opentelemetry-instrumentation-ollama>=0.40.8"]
+openai=["opentelemetry-instrumentation-openai>=0.40.8"]
+pinecone=["opentelemetry-instrumentation-pinecone>=0.40.8"]
+qdrant=["opentelemetry-instrumentation-qdrant>=0.40.8"]
+replicate=["opentelemetry-instrumentation-replicate>=0.40.8"]
+sagemaker=["opentelemetry-instrumentation-sagemaker>=0.40.8"]
+together=["opentelemetry-instrumentation-together>=0.40.8"]
+transformers=["opentelemetry-instrumentation-transformers>=0.40.8"]
+vertexai=["opentelemetry-instrumentation-vertexai>=0.40.8"]
+watsonx=["opentelemetry-instrumentation-watsonx>=0.40.8"]
+weaviate=["opentelemetry-instrumentation-weaviate>=0.40.8"]
+# `all` is the group added for convenience, if you want to install all
+# the instrumentations.
+# we suggest using package-manager-specific commands instead,
+# like `uv add lmnr --all-extras`
+all = [
+  "opentelemetry-instrumentation-alephalpha>=0.40.8",
+  "opentelemetry-instrumentation-anthropic>=0.40.8",
+  "opentelemetry-instrumentation-bedrock>=0.40.8",
+  "opentelemetry-instrumentation-chromadb>=0.40.8",
+  "opentelemetry-instrumentation-cohere>=0.40.8",
+  "opentelemetry-instrumentation-crewai>=0.40.8",
+  "opentelemetry-instrumentation-google-generativeai>=0.40.8",
+  "opentelemetry-instrumentation-groq>=0.40.8",
+  "opentelemetry-instrumentation-haystack>=0.40.8",
+  "opentelemetry-instrumentation-lancedb>=0.40.8",
+  "opentelemetry-instrumentation-langchain>=0.40.8",
+  "opentelemetry-instrumentation-llamaindex>=0.40.8",
+  "opentelemetry-instrumentation-marqo>=0.40.8",
+  "opentelemetry-instrumentation-mcp>=0.40.8",
+  "opentelemetry-instrumentation-milvus>=0.40.8",
+  "opentelemetry-instrumentation-mistralai>=0.40.8",
+  "opentelemetry-instrumentation-ollama>=0.40.8",
+  "opentelemetry-instrumentation-openai>=0.40.8",
+  "opentelemetry-instrumentation-pinecone>=0.40.8",
+  "opentelemetry-instrumentation-qdrant>=0.40.8",
+  "opentelemetry-instrumentation-replicate>=0.40.8",
+  "opentelemetry-instrumentation-sagemaker>=0.40.8",
+  "opentelemetry-instrumentation-together>=0.40.8",
+  "opentelemetry-instrumentation-transformers>=0.40.8",
+  "opentelemetry-instrumentation-vertexai>=0.40.8",
+  "opentelemetry-instrumentation-watsonx>=0.40.8",
+  "opentelemetry-instrumentation-weaviate>=0.40.8"
+]
+
+[dependency-groups]
+dev = [
+  "autopep8>=2.3.2",
+  "flake8>=7.2.0",
+  "pytest>=8.3.5",
+  "pytest-sugar>=1.0.0",
+  "pytest-asyncio>=0.26.0",
+  "playwright>=1.52.0",
+  "vcrpy>=7.0.0",
+  "openai>=1.77.0",
+  "pytest-recording>=0.13.4",
+  "patchright>=1.52.3",
+  "google-genai>=1.19.0",
+  "langgraph>=0.4.8",
+  "langchain-core>=0.3.64",
+  "langchain>=0.3.25",
+]
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.uv.workspace]
+members = ["examples/fastapi-app"]
+# we can move to uv_build, once it's more stable
+# https://github.com/astral-sh/uv/issues/3957
+# requires = ["uv_build>=0.6.16,<0.7"]
+# build-backend = "uv_build"

{lmnr-0.6.8 → lmnr-0.6.10}/src/lmnr/cli.py

@@ -1,6 +1,8 @@
 from argparse import ArgumentParser
 import asyncio
+import glob
 import importlib.util
+import json
 import os
 import re
 import sys
@@ -23,27 +25,27 @@ def add_cursor_rules():
     """Download laminar.mdc file from a hardcoded public URL and save it to .cursor/rules/laminar.mdc"""
     # Hardcoded URL for the laminar.mdc file
     url = "https://raw.githubusercontent.com/lmnr-ai/lmnr/dev/rules/laminar.mdc"
-    
+
     # Create .cursor/rules directory if it doesn't exist
     rules_dir = Path(".cursor/rules")
     rules_dir.mkdir(parents=True, exist_ok=True)
-    
+
     # Define the target file path
     target_file = rules_dir / "laminar.mdc"
-    
+
     try:
         LOG.info(f"Downloading laminar.mdc from {url}")
-        
+
         # Download the file
         with urllib.request.urlopen(url) as response:
             content = response.read()
-        
+
         # Write the content to the target file (this will overwrite if it exists)
-        with open(target_file, 'wb') as f:
+        with open(target_file, "wb") as f:
             f.write(content)
-        
+
         LOG.info(f"Successfully downloaded laminar.mdc to {target_file}")
-        
+
     except urllib.error.URLError as e:
         LOG.error(f"Failed to download file from {url}: {e}")
         sys.exit(1)
@@ -55,7 +57,7 @@ def add_cursor_rules():
 async def run_evaluation(args):
     sys.path.append(os.getcwd())
 
-    if args.file is None:
+    if len(args.file) == 0:
         files = [
             os.path.join(EVAL_DIR, f)
             for f in os.listdir(EVAL_DIR)
@@ -71,9 +73,17 @@ async def run_evaluation(args):
         LOG.info(f"Located {len(files)} evaluation files in {EVAL_DIR}")
 
     else:
-        files = [args.file]
+        files = []
+        for pattern in args.file:
+            matches = glob.glob(pattern)
+            if matches:
+                files.extend(matches)
+            else:
+                # If no matches found, treat as literal filename
+                files.append(pattern)
 
     prep_token = PREPARE_ONLY.set(True)
+    scores = []
     try:
         for file in files:
             LOG.info(f"Running evaluation from {file}")
@@ -83,33 +93,56 @@ async def run_evaluation(args):
             spec = importlib.util.spec_from_file_location(name, file)
             if spec is None or spec.loader is None:
                 LOG.error(f"Could not load module specification from {file}")
-                if args.fail_on_error:
-                    return
-                continue
+                if args.continue_on_error:
+                    continue
+                return
             mod = importlib.util.module_from_spec(spec)
             sys.modules[name] = mod
 
             spec.loader.exec_module(mod)
-            evaluations: list[Evaluation] | None = EVALUATION_INSTANCES.get()
-            if evaluations is None:
-                LOG.warning("Evaluation instance not found")
-                if args.fail_on_error:
-                    return
-                continue
+            evaluations = []
+            try:
+                evaluations: list[Evaluation] | None = EVALUATION_INSTANCES.get()
+                if evaluations is None:
+                    raise LookupError()
+            # may be raised by `get()` or manually by us above
+            except LookupError:
+                log_evaluation_instance_not_found()
+                if args.continue_on_error:
+                    continue
+                return
 
             LOG.info(f"Loaded {len(evaluations)} evaluations from {file}")
 
             for evaluation in evaluations:
                 try:
-                    await evaluation.run()
+                    eval_scores = await evaluation.run()
+                    scores.append(
+                        {
+                            "file": file,
+                            "scores": eval_scores,
+                        }
+                    )
                 except Exception as e:
                     LOG.error(f"Error running evaluation: {e}")
-                    if args.fail_on_error:
+                    if not args.continue_on_error:
                         raise
+
+        if args.output_file:
+            with open(args.output_file, "w") as f:
+                json.dump(scores, f, indent=2)
     finally:
         PREPARE_ONLY.reset(prep_token)
 
 
+def log_evaluation_instance_not_found():
+    LOG.warning(
+        "Evaluation instance not found. "
+        "`evaluate` must be called at the top level of the file, "
+        "not inside a function when running evaluations from the CLI."
+    )
+
+
 def cli():
     parser = ArgumentParser(
         prog="lmnr",
@@ -125,21 +158,27 @@ def cli():
     )
     parser_eval.add_argument(
         "file",
-        nargs="?",
-        help="A file containing the evaluation to run."
+        nargs="*",
+        help="Files or a file containing the evaluation to run."
         + "If no file name is provided, all evaluation files in the `evals` directory are run as long"
         + "as they match *_eval.py or eval_*.py",
-        default=None,
+        default=[],
     )
 
     parser_eval.add_argument(
-        "--fail-on-error",
+        "--continue-on-error",
         action="store_true",
         default=False,
-        help="Fail on error",
+        help="Continue execution upon errors",
+    )
+
+    parser_eval.add_argument(
+        "--output-file",
+        help="Output file to write the results to. Outputs are written in JSON format.",
+        nargs="?",
     )
 
-    parser_download = subparsers.add_parser(
+    subparsers.add_parser(
         "add-cursor-rules",
         description="Download laminar.mdc file and add it to .cursor/rules",
         help="Download laminar.mdc file and add it to .cursor/rules",

{lmnr-0.6.8 → lmnr-0.6.10}/src/lmnr/opentelemetry_lib/__init__.py

@@ -54,6 +54,8 @@ class TracerManager:
 
     @staticmethod
     def flush() -> bool:
+        if not hasattr(TracerManager, "_TracerManager__tracer_wrapper"):
+            return False
         return TracerManager.__tracer_wrapper.flush()
 
     @staticmethod

{lmnr-0.6.8 → lmnr-0.6.10}/src/lmnr/opentelemetry_lib/opentelemetry/instrumentation/google_genai/__init__.py

@@ -1,6 +1,7 @@
 """OpenTelemetry Google Generative AI API instrumentation"""
 
 from collections import defaultdict
+import json
 import logging
 import os
 from typing import AsyncGenerator, Callable, Collection, Generator
@@ -11,7 +12,9 @@ from .config import (
     Config,
 )
 from .utils import (
+    ProcessedContentPart,
     dont_throw,
+    get_content,
     role_from_content_union,
     set_span_attribute,
     process_content_union,
@@ -159,7 +162,7 @@ def _set_request_attributes(span, args, kwargs):
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
-                process_content_union(system_instruction),
+                (get_content(process_content_union(system_instruction)) or {}).get("text", ""),
             )
             set_span_attribute(
                 span, f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role", "system"
@@ -169,11 +172,42 @@ def _set_request_attributes(span, args, kwargs):
         if not isinstance(contents, list):
             contents = [contents]
         for content in contents:
+            processed_content = process_content_union(content)
+            content_str = get_content(processed_content)
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.content",
-                process_content_union(content),
+                (
+                    content_str
+                    if isinstance(content_str, str)
+                    else json.dumps(content_str)
+                ),
             )
+            blocks = (
+                processed_content
+                if isinstance(processed_content, list)
+                else [processed_content]
+            )
+            for j, block in enumerate(blocks):
+                block_dict = to_dict(block)
+                if not block_dict.get("function_call"):
+                    continue
+                function_call = to_dict(block_dict.get("function_call", {}))
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.name",
+                    function_call.get("name"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.id",
+                    function_call.get("id"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.tool_calls.{j}.arguments",
+                    json.dumps(function_call.get("arguments")),
+                )
             set_span_attribute(
                 span,
                 f"{gen_ai_attributes.GEN_AI_PROMPT}.{i}.role",
@@ -218,23 +252,64 @@ def _set_response_attributes(span, response: types.GenerateContentResponse):
         )
 
     if should_send_prompts():
-        if len(candidates) > 1:
-            for i, candidate in enumerate(candidates):
+        set_span_attribute(
+            span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.role", "model"
+        )
+        candidates_list = candidates if isinstance(candidates, list) else [candidates]
+        for i, candidate in enumerate(candidates_list):
+            processed_content = process_content_union(candidate.content)
+            if isinstance(processed_content, list):
+                if all(
+                    isinstance(item, dict) and item.get("type") == "text"
+                    for item in processed_content
+                ):
+                    content_str = processed_content[0]["text"]
+                elif all(
+                    isinstance(item, ProcessedContentPart) and item.content
+                    for item in processed_content
+                ):
+                    content_str = processed_content[0].content
+                else:
+                    content_str = get_content(processed_content)
+            else:
+                content_str = get_content(processed_content)
+            set_span_attribute(
+                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.role", "model"
+            )
+            set_span_attribute(
+                span,
+                f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.content",
+                (
+                    content_str
+                    if isinstance(content_str, str)
+                    else json.dumps(content_str)
+                ),
+            )
+            blocks = (
+                processed_content
+                if isinstance(processed_content, list)
+                else [processed_content]
+            )
+            for j, block in enumerate(blocks):
+                block_dict = to_dict(block)
+                if not block_dict.get("function_call"):
+                    continue
+                function_call = to_dict(block_dict.get("function_call", {}))
                 set_span_attribute(
                     span,
-                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.content",
-                    process_content_union(candidate.content),
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.name",
+                    function_call.get("name"),
                 )
                 set_span_attribute(
-                    span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.role", "assistant"
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.id",
+                    function_call.get("id"),
+                )
+                set_span_attribute(
+                    span,
+                    f"{gen_ai_attributes.GEN_AI_COMPLETION}.{i}.tool_calls.{j}.arguments",
+                    json.dumps(function_call.get("arguments")),
                 )
-        else:
-            set_span_attribute(
-                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.content", response.text
-            )
-            set_span_attribute(
-                span, f"{gen_ai_attributes.GEN_AI_COMPLETION}.0.role", "assistant"
-            )
 
 
 @dont_throw
@@ -433,7 +508,7 @@ class GoogleGenAiSdkInstrumentor(BaseInstrumentor):
 
     def _instrument(self, **kwargs):
         tracer_provider = kwargs.get("tracer_provider")
-        tracer = get_tracer(__name__, "0.0.1a0", tracer_provider)
+        tracer = get_tracer(__name__, "0.0.1a1", tracer_provider)
 
         for wrapped_method in WRAPPED_METHODS:
             wrap_function_wrapper(