lmnr 0.3.7__py3-none-any.whl → 0.4.1__py3-none-any.whl

lmnr-0.4.1.dist-info/METADATA

@@ -0,0 +1,214 @@
+Metadata-Version: 2.1
+Name: lmnr
+Version: 0.4.1
+Summary: Python SDK for Laminar AI
+License: Apache-2.0
+Author: lmnr.ai
+Requires-Python: >=3.9,<4.0
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: asyncio (>=3.4.3,<4.0.0)
+Requires-Dist: backoff (>=2.2.1,<3.0.0)
+Requires-Dist: pydantic (>=2.7.4,<3.0.0)
+Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
+Requires-Dist: requests (>=2.32.3,<3.0.0)
+Requires-Dist: traceloop-sdk (>=0.29.2,<0.30.0)
+Description-Content-Type: text/markdown
+
+# Laminar Python
+
+OpenTelemetry log sender for [Laminar](https://github.com/lmnr-ai/lmnr) for Python code.
+
+ <a href="https://pypi.org/project/lmnr/"> ![PyPI - Version](https://img.shields.io/pypi/v/lmnr?label=lmnr&logo=pypi&logoColor=3775A9) </a>
+![PyPI - Downloads](https://img.shields.io/pypi/dm/lmnr)
+![PyPI - Python Version](https://img.shields.io/pypi/pyversions/lmnr)
+
+
+
+## Quickstart
+```sh
+python3 -m venv .myenv
+source .myenv/bin/activate  # or use your favorite env management tool
+
+pip install lmnr
+```
+
+And the in your main Python file
+
+```python
+from lmnr import Laminar as L
+
+L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>")
+```
+
+This will automatically instrument most of the LLM, Vector DB, and related
+calls with OpenTelemetry-compatible instrumentation.
+
+We rely on the amazing [OpenLLMetry](https://github.com/traceloop/openllmetry), open-source package
+by TraceLoop, to achieve that.
+
+### Project API key
+
+Get the key from the settings page of your Laminar project ([Learn more](https://docs.lmnr.ai/api-reference/introduction#authentication)).
+You can either pass it to `.initialize()` or set it to `.env` at the root of your package with the key `LMNR_PROJECT_API_KEY`.
+
+## Instrumentation
+
+In addition to automatic instrumentation, we provide a simple `@observe()` decorator, if you want more fine-grained tracing
+or to trace other functions.
+
+### Example
+
+```python
+import os
+from openai import OpenAI
+
+
+from lmnr import observe, Laminar as L
+L.initialize(project_api_key="<LMNR_PROJECT_API_KEY>")
+
+client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+@observe()  # annotate all functions you want to trace
+def poem_writer(topic="turbulence"):
+    prompt = f"write a poem about {topic}"
+    response = client.chat.completions.create(
+        model="gpt-4o",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {"role": "user", "content": prompt},
+        ],
+    )
+    poem = response.choices[0].message.content
+    return poem
+
+print(poem_writer(topic="laminar flow"))
+```
+
+
+## Sending events
+
+You can send events in two ways:
+- `.event(name, value)` – for a pre-defined event with one of possible values.
+- `.evaluate_event(name, evaluator, data)` – for an event that is evaluated by evaluator pipeline based on the data.
+
+Note that to run an evaluate event, you need to crate an evaluator pipeline and create a target version for it.
+
+Read our [docs](https://docs.lmnr.ai) to learn more about event types and how they are created and evaluated.
+
+### Example
+
+```python
+from lmnr import Laminar as L
+# ...
+poem = response.choices[0].message.content
+
+# this will register True or False value with Laminar
+L.event("topic alignment", topic in poem)
+
+# this will run the pipeline `check_wordy` with `poem` set as the value
+# of `text_input` node, and write the result as an event with name
+# "excessive_wordiness"
+L.evaluate_event("excessive_wordiness", "check_wordy", {"text_input": poem})
+```
+
+## Laminar pipelines as prompt chain managers
+
+You can create Laminar pipelines in the UI and manage chains of LLM calls there.
+
+After you are ready to use your pipeline in your code, deploy it in Laminar by selecting the target version for the pipeline.
+
+Once your pipeline target is set, you can call it from Python in just a few lines.
+
+Example use:
+
+```python
+from lmnr import Laminar as L
+
+L.initialize('<YOUR_PROJECT_API_KEY>')
+
+result = l.run(
+    pipeline = 'my_pipeline_name',
+    inputs = {'input_node_name': 'some_value'},
+    # all environment variables
+    env = {'OPENAI_API_KEY': 'sk-some-key'},
+)
+```
+
+Resulting in:
+
+```python
+>>> result
+PipelineRunResponse(
+    outputs={'output': {'value': [ChatMessage(role='user', content='hello')]}},
+    # useful to locate your trace
+    run_id='53b012d5-5759-48a6-a9c5-0011610e3669'
+)
+```
+
+## Running offline evaluations on your data
+
+You can evaluate your code with your own data and send it to Laminar using the `Evaluation` class.
+
+Evaluation takes in the following parameters:
+- `name` – the name of your evaluation. If no such evaluation exists in the project, it will be created. Otherwise, data will be pushed to the existing evaluation
+- `data` – an array of `EvaluationDatapoint` objects, where each `EvaluationDatapoint` has two keys: `target` and `data`, each containing a key-value object. Alternatively, you can pass in dictionaries, and we will instantiate `EvaluationDatapoint`s with pydantic if possible
+- `executor` – the logic you want to evaluate. This function must take `data` as the first argument, and produce any output. *
+- `evaluators` – evaluaton logic. List of functions that take output of executor as the first argument, `target` as the second argument and produce a numeric scores. Each function can produce either a single number or `dict[str, int|float]` of scores.
+
+\* If you already have the outputs of executors you want to evaluate, you can specify the executor as an identity function, that takes in `data` and returns only needed value(s) from it.
+
+### Example
+
+```python
+from openai import AsyncOpenAI
+import asyncio
+import os
+
+openai_client = AsyncOpenAI(api_key=os.environ["OPENAI_API_KEY"])
+
+async def get_capital(data):
+    country = data["country"]
+    response = await openai_client.chat.completions.create(
+        model="gpt-4o-mini",
+        messages=[
+            {"role": "system", "content": "You are a helpful assistant."},
+            {
+                "role": "user",
+                "content": f"What is the capital of {country}? Just name the "
+                "city and nothing else",
+            },
+        ],
+    )
+    return response.choices[0].message.content.strip()
+
+
+# Evaluation data
+data = [
+    {"data": {"country": "Canada"}, "target": {"capital": "Ottawa"}},
+    {"data": {"country": "Germany"}, "target": {"capital": "Berlin"}},
+    {"data": {"country": "Tanzania"}, "target": {"capital": "Dodoma"}},
+]
+
+
+def evaluator_A(output, target):
+    return 1 if output == target["capital"] else 0
+
+
+# Create an Evaluation instance
+e = Evaluation(
+    name="py-evaluation-async",
+    data=data,
+    executor=get_capital,
+    evaluators=[evaluator_A],
+    project_api_key=os.environ["LMNR_PROJECT_API_KEY"],
+)
+
+# Run the evaluation
+asyncio.run(e.run())
+```
+

lmnr-0.4.1.dist-info/RECORD

@@ -0,0 +1,13 @@
+lmnr/__init__.py,sha256=wQwnHl662Xcz7GdSofFsEjmAK0nxioYA2Yq6Q78m4ps,194
+lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+lmnr/sdk/decorators.py,sha256=Xs6n0TGX9LZ9i1hE_UZz4LEyd_ZAfpVGfNQh_rKwOuA,2493
+lmnr/sdk/evaluations.py,sha256=LkQApHAhR7y_rC2ovnJi8yHpdcl0-7yesdBqvOJ0BKg,6107
+lmnr/sdk/laminar.py,sha256=970fvaw969pBdBqrDRD8lQ82uPEn8V5n-4rIIe_5pqM,16552
+lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
+lmnr/sdk/types.py,sha256=gDwRSWR9A1__FGtQhVaFc6PUYQuIhubo5tpfYAajTQQ,4055
+lmnr/sdk/utils.py,sha256=ZsGJ86tq8lIbvOhSb1gJWH5K3GylO_lgX68FN6rG2nM,3358
+lmnr-0.4.1.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
+lmnr-0.4.1.dist-info/METADATA,sha256=_g6TaAlJuPxl_sbWYPjlg4380V0Ja34P8OXHAXofakI,7025
+lmnr-0.4.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+lmnr-0.4.1.dist-info/entry_points.txt,sha256=Qg7ZRax4k-rcQsZ26XRYQ8YFSBiyY2PNxYfq4a6PYXI,41
+lmnr-0.4.1.dist-info/RECORD,,

lmnr/sdk/client.py

@@ -1,161 +0,0 @@
-from .tracing_types import Span, Trace
-
-from pydantic.alias_generators import to_snake
-from typing import Any, Optional, Union
-import dotenv
-import json
-import logging
-import os
-import requests
-import uuid
-
-from .types import (
-    PipelineRunError,
-    PipelineRunResponse,
-    NodeInput,
-    PipelineRunRequest,
-)
-
-
-class APIError(Exception):
-    def __init__(self, status: Union[int, str], message: str, details: Any = None):
-        self.message = message
-        self.status = status
-        self.details = details
-
-    def __str__(self):
-        msg = "{0} ({1}): {2}"
-        return msg.format(self.message, self.status, self.details)
-
-
-class Laminar:
-    _base_url = "https://api.lmnr.ai"
-
-    def __init__(self, project_api_key: Optional[str] = None):
-        self.project_api_key = project_api_key or os.environ.get("LMNR_PROJECT_API_KEY")
-        if not self.project_api_key:
-            dotenv_path = dotenv.find_dotenv(usecwd=True)
-            self.project_api_key = dotenv.get_key(
-                dotenv_path=dotenv_path, key_to_get="LMNR_PROJECT_API_KEY"
-            )
-        if not self.project_api_key:
-            raise ValueError(
-                "Please initialize the Laminar object with your project API key or set "
-                "the LMNR_PROJECT_API_KEY environment variable in your environment or .env file"
-            )
-
-    def run(
-        self,
-        pipeline: str,
-        inputs: dict[str, NodeInput],
-        env: dict[str, str] = {},
-        metadata: dict[str, str] = {},
-        parent_span_id: Optional[uuid.UUID] = None,
-        trace_id: Optional[uuid.UUID] = None,
-    ) -> PipelineRunResponse:
-        """Runs the pipeline with the given inputs
-
-        Args:
-            pipeline (str): name of the Laminar pipeline
-            inputs (dict[str, NodeInput]):
-                inputs to the endpoint's target pipeline.
-                Keys in the dictionary must match input node names
-            env (dict[str, str], optional):
-                Environment variables for the pipeline execution.
-                Defaults to {}.
-            metadata (dict[str, str], optional):
-                any custom metadata to be stored
-                with execution trace. Defaults to {}.
-            parent_span_id (Optional[uuid.UUID], optional):
-                parent span id for the resulting span.
-                Must usually be SpanContext.id()
-                Defaults to None.
-            trace_id (Optional[uuid.UUID], optional):
-                trace id for the resulting trace.
-                Must usually be TraceContext.id()
-                Defaults to None.
-
-        Returns:
-            PipelineRunResponse: response object containing the outputs
-
-        Raises:
-            ValueError: if project API key is not set
-            PipelineRunError: if the endpoint run fails
-        """
-        if self.project_api_key is None:
-            raise ValueError(
-                "Please initialize the Laminar object with your project API key or set "
-                "the LMNR_PROJECT_API_KEY environment variable"
-            )
-        try:
-            request = PipelineRunRequest(
-                inputs=inputs,
-                pipeline=pipeline,
-                env=env,
-                metadata=metadata,
-                parent_span_id=parent_span_id,
-                trace_id=trace_id,
-            )
-        except Exception as e:
-            raise ValueError(f"Invalid request: {e}")
-
-        response = requests.post(
-            self._base_url + "/v1/pipeline/run",
-            data=json.dumps(request.to_dict()),
-            headers=self._headers(),
-        )
-        if response.status_code != 200:
-            raise PipelineRunError(response)
-        try:
-            resp_json = response.json()
-            keys = list(resp_json.keys())
-            for key in keys:
-                value = resp_json[key]
-                del resp_json[key]
-                resp_json[to_snake(key)] = value
-            return PipelineRunResponse(**resp_json)
-        except Exception:
-            raise PipelineRunError(response)
-
-    def batch_post_traces(self, batch: list[Union[Span, Trace]]):
-        log = logging.getLogger("laminar.client")
-        url = self._base_url + "/v1/observations"
-        data = json.dumps({"observations": [item.to_dict() for item in batch]})
-        log.debug(f"making request to {url}")
-        headers = self._headers()
-        res = requests.post(url, data=data, headers=headers)
-
-        if res.status_code == 200:
-            log.debug("data uploaded successfully")
-
-        return self._process_response(
-            res, success_message="data uploaded successfully", return_json=False
-        )
-
-    def _process_response(
-        self, res: requests.Response, success_message: str, return_json: bool = True
-    ) -> Union[requests.Response, Any]:
-        log = logging.getLogger("laminar.client")
-        log.debug("received response: %s", res.text)
-        if res.status_code in (200, 201):
-            log.debug(success_message)
-            if return_json:
-                try:
-                    return res.json()
-                except json.JSONDecodeError:
-                    log.error("Response is not valid JSON.")
-                    raise APIError(res.status_code, "Invalid JSON response received")
-            else:
-                return res
-        try:
-            payload = res.json()
-            log.error("received error response: %s", payload)
-            raise APIError(res.status_code, payload)
-        except (KeyError, ValueError):
-            raise APIError(res.status_code, res.text)
-
-    def _headers(self):
-        return {
-            "Authorization": "Bearer " + self.project_api_key,
-            "Content-Type": "application/json",
-        }

lmnr/sdk/collector.py

@@ -1,177 +0,0 @@
-from .client import APIError, Laminar
-from .tracing_types import Span, Trace
-
-from queue import Queue, Empty, Full
-from typing import Union
-
-import atexit
-import backoff
-import logging
-import time
-import threading
-
-
-class Collector(threading.Thread):
-    _log = logging.getLogger("laminar.collector")
-    _queue: Queue[Union[Span, Trace]]
-    _client: Laminar
-    _flush_interval: float
-
-    def __init__(
-        self,
-        queue: Queue[Union[Span, Trace]],
-        client: Laminar,
-        flush_interval: float = 5.0,
-    ):
-        super().__init__()
-        self.daemon = True
-        self._queue = queue
-        self.running = True
-        self._flush_interval = flush_interval
-        self._client = client
-
-    def run(self):
-        """Runs the collector."""
-        self._log.debug("collector is running...")
-        while self.running:
-            self.upload()
-
-    def upload(self):
-        """Upload the next batch of items, return whether successful."""
-        batch = self._next()
-        if len(batch) == 0:
-            return
-
-        try:
-            self._upload_batch(batch)
-        except Exception as e:
-            self._log.exception("error uploading: %s", e)
-        finally:
-            # mark items as acknowledged from queue
-            for _ in batch:
-                self._queue.task_done()
-
-    def pause(self):
-        self.running = False
-
-    def _next(self):
-        items = []
-        start_time = time.monotonic()
-
-        while True:
-            elapsed = time.monotonic() - start_time
-            if elapsed >= self._flush_interval:
-                break
-            try:
-                item = self._queue.get(
-                    block=True, timeout=self._flush_interval - elapsed
-                )
-                items.append(item)
-            except Empty:
-                break
-        return items
-
-    def _upload_batch(self, batch: list[Union[Trace, Span]]):
-        self._log.debug("uploading batch of %d items", len(batch))
-
-        @backoff.on_exception(backoff.expo, Exception, max_tries=5)
-        def execute_task_with_backoff(batch: list[Union[Trace, Span]]):
-            try:
-                self._client.batch_post_traces(batch=batch)
-            except Exception as e:
-                if (
-                    isinstance(e, APIError)
-                    and 400 <= int(e.status) < 500
-                    and int(e.status) != 429
-                ):
-                    self._log.warn(
-                        f"Received {e.status} error by Laminar server, not retrying: {e.message}"
-                    )
-                    return
-
-                raise e
-
-        execute_task_with_backoff(batch)
-        self._log.debug("successfully uploaded batch of %d items", len(batch))
-
-
-class ThreadManager:
-    _log = logging.getLogger("laminar.task_manager")
-    _queue: Queue[Union[Span, Trace]]
-    _client: Laminar
-    _max_task_queue_size: int
-    _flush_interval: float
-    _collectors: list[Collector] = []
-    _threads: int
-
-    def __init__(
-        self,
-        client: Laminar,
-        flush_interval: float = 2.0,
-        max_task_queue_size: int = 1000,
-        threads: int = 1,
-    ):
-        self._max_task_queue_size = max_task_queue_size
-        self._threads = threads
-        self._queue = Queue(maxsize=self._max_task_queue_size)
-        self._flush_interval = flush_interval
-        self._client = client
-        for _ in range(self._threads):
-            collector = Collector(
-                queue=self._queue,
-                client=self._client,
-                flush_interval=flush_interval,
-            )
-            self._collectors.append(collector)
-            collector.start()
-        atexit.register(self.join)
-
-    def add_task(self, event: Union[Span, Trace]) -> bool:
-        try:
-            self._queue.put(event, block=False)
-            return True
-        except Full:
-            self._log.warning("queue is full")
-            return False
-        except Exception as e:
-            self._log.exception(f"Exception in adding task {e}")
-
-            return False
-
-    def flush(self):
-        """Forces a flush from the internal queue to the server"""
-        self._log.debug("flushing queue")
-        queue = self._queue
-        size = queue.qsize()
-        queue.join()
-        # Note that this message may not be precise, because of threading.
-        self._log.debug("successfully flushed about %s items.", size)
-
-    def join(self):
-        """Ends the collector threads once the queue is empty.
-        Blocks execution until finished
-        """
-        self._log.debug(f"joining {len(self._collectors)} collector threads")
-
-        # pause all collectors before joining them so we don't have to wait for multiple
-        # flush intervals to join them all.
-        for collector in self._collectors:
-            collector.pause()
-
-        for i, collector in enumerate(self._collectors):
-            try:
-                collector.join()
-            except RuntimeError:
-                # collector thread has not started
-                pass
-
-            self._log.debug(f"collector thread {i} joined")
-
-    def shutdown(self):
-        """Flush all messages and cleanly shutdown the client"""
-        self._log.debug("shutdown initiated")
-
-        self.flush()
-        self.join()
-
-        self._log.debug("shutdown completed")

lmnr/sdk/constants.py

@@ -1 +0,0 @@
-CURRENT_TRACING_VERSION = "0.1.0"