lmnr 0.4.10__tar.gz → 0.4.12b1__tar.gz

{lmnr-0.4.10 → lmnr-0.4.12b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.10
+Version: 0.4.12b1
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -11,6 +11,7 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: argparse (>=1.0,<2.0)
 Requires-Dist: asyncio (>=3.0,<4.0)
 Requires-Dist: backoff (>=2.0,<3.0)
 Requires-Dist: colorama (>=0.4,<0.5)
@@ -54,6 +55,7 @@ Requires-Dist: pydantic (>=2.7,<3.0)
 Requires-Dist: python-dotenv (>=1.0,<2.0)
 Requires-Dist: requests (>=2.0,<3.0)
 Requires-Dist: tenacity (>=8.0,<9.0)
+Requires-Dist: tqdm (>=4.0,<5.0)
 Description-Content-Type: text/markdown
 
 # Laminar Python
@@ -67,6 +69,9 @@ OpenTelemetry log sender for [Laminar](https://github.com/lmnr-ai/lmnr) for Pyth
 
 
 ## Quickstart
+
+First, install the package:
+
 ```sh
 python3 -m venv .myenv
 source .myenv/bin/activate  # or use your favorite env management tool
@@ -74,22 +79,39 @@ source .myenv/bin/activate  # or use your favorite env management tool
 pip install lmnr
 ```
 
-And the in your main Python file
+Then, you can initialize Laminar in your main file and instrument your code.
 
 ```python
-from lmnr import Laminar as L, Instruments
+import os
+from openai import OpenAI
+from lmnr import Laminar as L
 
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
-```
+L.initialize(
+    project_api_key=os.environ["LMNR_PROJECT_API_KEY"],
+)
 
-If you want to automatically instrument particular LLM, Vector DB, and related
-calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+def poem_writer(topic: str):
+    prompt = f"write a poem about {topic}"
+
+    # OpenAI calls are automatically instrumented
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": prompt},
+        ],
+    )
+    poem = response.choices[0].message.content
+    return poem
 
-You can pass an empty set as `instruments=set()` to disable any kind of automatic instrumentation. 
-Also if you want to automatically instrument all supported libraries, then pass `instruments=None` or don't pass `instruments` at all.
+if __name__ == "__main__":
+    print(poem_writer("laminar flow"))
+
+```
 
-Our code is based on the [OpenLLMetry](https://github.com/traceloop/openllmetry), open-source package
-by TraceLoop. Also, we are grateful to Traceloop for implementing autoinstrumentations for many libraries.
+Note that you need to only initialize Laminar once in your application.
 
 ### Project API key
 
@@ -98,67 +120,84 @@ You can either pass it to `.initialize()` or set it to `.env` at the root of you
 
 ## Instrumentation
 
-In addition to automatic instrumentation, we provide a simple `@observe()` decorator, if you want more fine-grained tracing
-or to trace other functions.
+### Manual instrumentation
 
-### Example
+To instrument any function in your code, we provide a simple `@observe()` decorator.
+This can be useful if you want to trace a request handler or a function which combines multiple LLM calls.
 
 ```python
 import os
 from openai import OpenAI
+from lmnr import Laminar as L, Instruments
 
-
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])
 
 client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
-@observe()  # annotate all functions you want to trace
-def poem_writer(topic="turbulence"):
+def poem_writer(topic: str):
     prompt = f"write a poem about {topic}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt},
+    ]
+
+    # OpenAI calls are still automatically instrumented
     response = client.chat.completions.create(
         model="gpt-4o",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": prompt},
-        ],
+        messages=messages,
     )
     poem = response.choices[0].message.content
+
     return poem
 
-print(poem_writer(topic="laminar flow"))
+@observe()
+def generate_poems():
+    poem1 = poem_writer(topic="laminar flow")
+    L.event("is_poem_generated", True)
+    poem2 = poem_writer(topic="turbulence")
+    L.event("is_poem_generated", True)
+    poems = f"{poem1}\n\n---\n\n{poem2}"
+    return poems
 ```
 
-### Manual instrumentation
-
-Also, you can `Laminar.start_as_current_span` if you want to record a chunk of your code.
+Also, you can use `Laminar.start_as_current_span` if you want to record a chunk of your code using `with` statement.
 
 ```python
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+def handle_user_request(topic: str):
+    with L.start_as_current_span(name="poem_writer", input=topic):
+        ...
+
+        poem = poem_writer(topic=topic)
+        
+        ...
+        
+        # while within the span, you can attach laminar events to it
+        L.event("is_poem_generated", True)
 
-def poem_writer(topic="turbulence"):
-    prompt = f"write a poem about {topic}"
-    messages = [
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": prompt},
-    ]
+        # Use set_span_output to record the output of the span
+        L.set_span_output(poem)
+```
 
-    with L.start_as_current_span(name="poem_writer", input=messages):
-        # OpenAI calls are still automatically instrumented with OpenLLMetry
-        response = client.chat.completions.create(
-            model="gpt-4o",
-            messages=messages,
-        )
-        poem = response.choices[0].message.content
-        # while within the span, you can attach laminar events to it
-        L.event("event_name", "event_value")
+### Automatic instrumentation
 
-        L.set_span_output(poem) # set an output
+Laminar allows you to automatically instrument majority of the most popular LLM, Vector DB, database, requests, and other libraries.
 
-        return poem
+If you want to automatically instrument a default set of libraries, then simply do NOT pass `instruments` argument to `.initialize()`.
+See the full list of available instrumentations in the [enum](/src/lmnr/traceloop_sdk/instruments.py).
+
+If you want to automatically instrument only specific LLM, Vector DB, or other
+calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+For example, if you want to only instrument OpenAI and Anthropic, then do the following:
+
+```python
+from lmnr import Laminar as L, Instruments
+
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"], instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
 ```
 
+If you want to fully disable any kind of autoinstrumentation, pass an empty set as `instruments=set()` to `.initialize()`. 
+
+Majority of the autoinstrumentations are provided by Traceloop's [OpenLLMetry](https://github.com/traceloop/openllmetry).
 
 ## Sending events
 

{lmnr-0.4.10 → lmnr-0.4.12b1}/README.md

@@ -9,6 +9,9 @@ OpenTelemetry log sender for [Laminar](https://github.com/lmnr-ai/lmnr) for Pyth
 
 
 ## Quickstart
+
+First, install the package:
+
 ```sh
 python3 -m venv .myenv
 source .myenv/bin/activate  # or use your favorite env management tool
@@ -16,22 +19,39 @@ source .myenv/bin/activate  # or use your favorite env management tool
 pip install lmnr
 ```
 
-And the in your main Python file
+Then, you can initialize Laminar in your main file and instrument your code.
 
 ```python
-from lmnr import Laminar as L, Instruments
+import os
+from openai import OpenAI
+from lmnr import Laminar as L
 
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
-```
+L.initialize(
+    project_api_key=os.environ["LMNR_PROJECT_API_KEY"],
+)
 
-If you want to automatically instrument particular LLM, Vector DB, and related
-calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+def poem_writer(topic: str):
+    prompt = f"write a poem about {topic}"
+
+    # OpenAI calls are automatically instrumented
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": prompt},
+        ],
+    )
+    poem = response.choices[0].message.content
+    return poem
 
-You can pass an empty set as `instruments=set()` to disable any kind of automatic instrumentation. 
-Also if you want to automatically instrument all supported libraries, then pass `instruments=None` or don't pass `instruments` at all.
+if __name__ == "__main__":
+    print(poem_writer("laminar flow"))
+
+```
 
-Our code is based on the [OpenLLMetry](https://github.com/traceloop/openllmetry), open-source package
-by TraceLoop. Also, we are grateful to Traceloop for implementing autoinstrumentations for many libraries.
+Note that you need to only initialize Laminar once in your application.
 
 ### Project API key
 
@@ -40,67 +60,84 @@ You can either pass it to `.initialize()` or set it to `.env` at the root of you
 
 ## Instrumentation
 
-In addition to automatic instrumentation, we provide a simple `@observe()` decorator, if you want more fine-grained tracing
-or to trace other functions.
+### Manual instrumentation
 
-### Example
+To instrument any function in your code, we provide a simple `@observe()` decorator.
+This can be useful if you want to trace a request handler or a function which combines multiple LLM calls.
 
 ```python
 import os
 from openai import OpenAI
+from lmnr import Laminar as L, Instruments
 
-
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])
 
 client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
 
-@observe()  # annotate all functions you want to trace
-def poem_writer(topic="turbulence"):
+def poem_writer(topic: str):
     prompt = f"write a poem about {topic}"
+    messages = [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt},
+    ]
+
+    # OpenAI calls are still automatically instrumented
     response = client.chat.completions.create(
         model="gpt-4o",
-        messages=[
-            {"role": "system", "content": "You are a helpful assistant."},
-            {"role": "user", "content": prompt},
-        ],
+        messages=messages,
     )
     poem = response.choices[0].message.content
+
     return poem
 
-print(poem_writer(topic="laminar flow"))
+@observe()
+def generate_poems():
+    poem1 = poem_writer(topic="laminar flow")
+    L.event("is_poem_generated", True)
+    poem2 = poem_writer(topic="turbulence")
+    L.event("is_poem_generated", True)
+    poems = f"{poem1}\n\n---\n\n{poem2}"
+    return poems
 ```
 
-### Manual instrumentation
-
-Also, you can `Laminar.start_as_current_span` if you want to record a chunk of your code.
+Also, you can use `Laminar.start_as_current_span` if you want to record a chunk of your code using `with` statement.
 
 ```python
-from lmnr import observe, Laminar as L, Instruments
-L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>", instruments={Instruments.OPENAI})
+def handle_user_request(topic: str):
+    with L.start_as_current_span(name="poem_writer", input=topic):
+        ...
+
+        poem = poem_writer(topic=topic)
+        
+        ...
+        
+        # while within the span, you can attach laminar events to it
+        L.event("is_poem_generated", True)
 
-def poem_writer(topic="turbulence"):
-    prompt = f"write a poem about {topic}"
-    messages = [
-        {"role": "system", "content": "You are a helpful assistant."},
-        {"role": "user", "content": prompt},
-    ]
+        # Use set_span_output to record the output of the span
+        L.set_span_output(poem)
+```
 
-    with L.start_as_current_span(name="poem_writer", input=messages):
-        # OpenAI calls are still automatically instrumented with OpenLLMetry
-        response = client.chat.completions.create(
-            model="gpt-4o",
-            messages=messages,
-        )
-        poem = response.choices[0].message.content
-        # while within the span, you can attach laminar events to it
-        L.event("event_name", "event_value")
+### Automatic instrumentation
 
-        L.set_span_output(poem) # set an output
+Laminar allows you to automatically instrument majority of the most popular LLM, Vector DB, database, requests, and other libraries.
 
-        return poem
+If you want to automatically instrument a default set of libraries, then simply do NOT pass `instruments` argument to `.initialize()`.
+See the full list of available instrumentations in the [enum](/src/lmnr/traceloop_sdk/instruments.py).
+
+If you want to automatically instrument only specific LLM, Vector DB, or other
+calls with OpenTelemetry-compatible instrumentation, then pass the appropriate instruments to `.initialize()`.
+For example, if you want to only instrument OpenAI and Anthropic, then do the following:
+
+```python
+from lmnr import Laminar as L, Instruments
+
+L.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"], instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
 ```
 
+If you want to fully disable any kind of autoinstrumentation, pass an empty set as `instruments=set()` to `.initialize()`. 
+
+Majority of the autoinstrumentations are provided by Traceloop's [OpenLLMetry](https://github.com/traceloop/openllmetry).
 
 ## Sending events
 

{lmnr-0.4.10 → lmnr-0.4.12b1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "lmnr"
-version = "0.4.10"
+version = "0.4.12b1"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -11,7 +11,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.10"
+version = "0.4.12b1"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"
@@ -62,6 +62,8 @@ opentelemetry-instrumentation-weaviate = "^0.30.0"
 opentelemetry-instrumentation-alephalpha = "^0.30.0"
 opentelemetry-instrumentation-marqo = "^0.30.0"
 opentelemetry-instrumentation-groq = "^0.30.0"
+tqdm = "~=4.0"
+argparse = "~=1.0"
 
 [tool.poetry.group.dev.dependencies]
 autopep8 = "^2.2.0"
@@ -83,11 +85,8 @@ langchain-openai = "^0.1.15"
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"
 
-[project.entry-points.console_scripts]
-lmnr = "lmnr.cli.cli:cli"
-
 [tool.poetry.scripts]
-lmnr = "lmnr.cli.cli:cli"
+lmnr = "lmnr.cli:cli"
 
 [project.optional-dependencies]
 test = ["pytest"]

{lmnr-0.4.10 → lmnr-0.4.12b1}/src/lmnr/__init__.py

@@ -1,4 +1,4 @@
-from .sdk.evaluations import Evaluation
+from .sdk.evaluations import evaluate
 from .sdk.laminar import Laminar
 from .sdk.types import ChatMessage, PipelineRunError, PipelineRunResponse, NodeInput
 from .sdk.decorators import observe

lmnr-0.4.12b1/src/lmnr/cli.py

@@ -0,0 +1,39 @@
+from argparse import ArgumentParser
+import asyncio
+import importlib
+import os
+import sys
+
+from lmnr.sdk.evaluations import set_global_evaluation
+
+
+# TODO: Refactor this code
+async def run_evaluation(args):
+    sys.path.insert(0, os.getcwd())
+
+    with set_global_evaluation(True):
+        file = os.path.abspath(args.file)
+
+        spec = importlib.util.spec_from_file_location("run_eval", file)
+        mod = importlib.util.module_from_spec(spec)
+        spec.loader.exec_module(mod)
+
+        from lmnr.sdk.evaluations import _evaluation
+        evaluation = _evaluation
+        await evaluation.run()
+
+
+def cli():
+    parser = ArgumentParser(
+        prog="lmnr",
+        description="CLI for Laminar",
+    )
+
+    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
+
+    parser_eval = subparsers.add_parser("eval", description="Run an evaluation")
+    parser_eval.add_argument("file", help="A file containing the evaluation to run")
+    parser_eval.set_defaults(func=run_evaluation)
+
+    parsed = parser.parse_args()
+    asyncio.run(parsed.func(parsed))

{lmnr-0.4.10 → lmnr-0.4.12b1}/src/lmnr/sdk/decorators.py

@@ -4,22 +4,19 @@ from lmnr.traceloop_sdk.decorators.base import (
 )
 from opentelemetry.trace import INVALID_SPAN, get_current_span
 
-from typing import Callable, Optional, ParamSpec, TypeVar, cast
+from typing import Callable, Optional, cast
 
 from lmnr.traceloop_sdk.tracing.tracing import update_association_properties
 
 from .utils import is_async
 
-P = ParamSpec("P")
-R = TypeVar("R")
-
 
 def observe(
     *,
     name: Optional[str] = None,
     user_id: Optional[str] = None,
     session_id: Optional[str] = None,
-) -> Callable[[Callable[P, R]], Callable[P, R]]:
+) -> Callable[[Callable], Callable]:
     """The main decorator entrypoint for Laminar. This is used to wrap
     functions and methods to create spans.
 
@@ -41,7 +38,7 @@ def observe(
         R: Returns the result of the wrapped function
     """
 
-    def decorator(func: Callable[P, R]) -> Callable[P, R]:
+    def decorator(func: Callable) -> Callable:
         current_span = get_current_span()
         if current_span != INVALID_SPAN:
             if session_id is not None:
@@ -64,4 +61,4 @@ def observe(
             else entity_method(name=name)(func)
         )
 
-    return cast(Callable[P, R], decorator)
+    return cast(Callable, decorator)