lmnr 0.4.40__tar.gz → 0.4.42__tar.gz

{lmnr-0.4.40 → lmnr-0.4.42}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: lmnr
-Version: 0.4.40
+Version: 0.4.42
 Summary: Python SDK for Laminar AI
 License: Apache-2.0
 Author: lmnr.ai
@@ -38,6 +38,7 @@ Provides-Extra: transformers
 Provides-Extra: vertexai
 Provides-Extra: watsonx
 Provides-Extra: weaviate
+Requires-Dist: aiohttp (>=3.0,<4.0)
 Requires-Dist: argparse (>=1.0,<2.0)
 Requires-Dist: backoff (>=2.0,<3.0)
 Requires-Dist: deprecated (>=1.0,<2.0)
@@ -290,3 +291,15 @@ PipelineRunResponse(
 )
 ```
 
+## Semantic search
+
+You can perform a semantic search on a dataset in Laminar by calling `Laminar.semantic_search`.
+
+```python
+response = Laminar.semantic_search(
+    query="Greatest Chinese architectural wonders",
+    dataset_id=uuid.UUID("413f8404-724c-4aa4-af16-714d84fd7958"),
+)
+```
+
+[Read more](https://docs.lmnr.ai/datasets/indexing) about indexing and semantic search.

{lmnr-0.4.40 → lmnr-0.4.42}/README.md

@@ -204,3 +204,16 @@ PipelineRunResponse(
     run_id='53b012d5-5759-48a6-a9c5-0011610e3669'
 )
 ```
+
+## Semantic search
+
+You can perform a semantic search on a dataset in Laminar by calling `Laminar.semantic_search`.
+
+```python
+response = Laminar.semantic_search(
+    query="Greatest Chinese architectural wonders",
+    dataset_id=uuid.UUID("413f8404-724c-4aa4-af16-714d84fd7958"),
+)
+```
+
+[Read more](https://docs.lmnr.ai/datasets/indexing) about indexing and semantic search.

{lmnr-0.4.40 → lmnr-0.4.42}/pyproject.toml

@@ -6,7 +6,7 @@
 
 [project]
 name = "lmnr"
-version = "0.4.40"
+version = "0.4.42"
 description = "Python SDK for Laminar AI"
 authors = [
   { name = "lmnr.ai", email = "founders@lmnr.ai" }
@@ -17,7 +17,7 @@ license = "Apache-2.0"
 
 [tool.poetry]
 name = "lmnr"
-version = "0.4.40"
+version = "0.4.42"
 description = "Python SDK for Laminar AI"
 authors = ["lmnr.ai"]
 readme = "README.md"
@@ -43,6 +43,7 @@ jinja2 = "~=3.0"
 deprecated = "~=1.0"
 tqdm = "~=4.0"
 argparse = "~=1.0"
+aiohttp = "~=3.0"
 opentelemetry-instrumentation-alephalpha = {version = ">=0.33.12", optional = true}
 opentelemetry-instrumentation-anthropic = {version = ">=0.33.12", optional = true}
 opentelemetry-instrumentation-bedrock = {version = ">=0.33.12", optional = true}
@@ -69,6 +70,7 @@ opentelemetry-instrumentation-vertexai = {version = ">=0.33.12", optional = true
 opentelemetry-instrumentation-watsonx = {version = ">=0.33.12", optional = true}
 opentelemetry-instrumentation-weaviate = {version = ">=0.33.12", optional = true}
 
+[tool.poetry.extras]
 # List of all possible extras. You can specify one or more of these extras
 # when installing the package, using any of the following:
 # `pip install 'lmnr[anthropic,openai]'`
@@ -78,7 +80,6 @@ opentelemetry-instrumentation-weaviate = {version = ">=0.33.12", optional = true
 
 # `all` is the group added for convenience, if you want to install all
 # the instrumentations.
-[tool.poetry.extras]
 all = [
   "opentelemetry-instrumentation-alephalpha",
   "opentelemetry-instrumentation-anthropic",

lmnr-0.4.42/src/lmnr/cli.py

@@ -0,0 +1,53 @@
+from argparse import ArgumentParser
+import asyncio
+import importlib.util
+import os
+import sys
+
+from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
+
+
+async def run_evaluation(args):
+    sys.path.append(os.getcwd())
+
+    prep_token = PREPARE_ONLY.set(True)
+    try:
+        file = os.path.abspath(args.file)
+        name = "user_module"
+
+        spec = importlib.util.spec_from_file_location(name, file)
+        if spec is None or spec.loader is None:
+            raise ImportError(f"Could not load module specification from {file}")
+        mod = importlib.util.module_from_spec(spec)
+        sys.modules[name] = mod
+
+        spec.loader.exec_module(mod)
+        evaluation = EVALUATION_INSTANCE.get()
+        if evaluation is None:
+            raise RuntimeError("Evaluation instance not found")
+
+        await evaluation.run()
+    finally:
+        PREPARE_ONLY.reset(prep_token)
+
+
+def cli():
+    parser = ArgumentParser(
+        prog="lmnr",
+        description="CLI for Laminar",
+    )
+
+    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
+
+    parser_eval = subparsers.add_parser(
+        "eval",
+        description="Run an evaluation",
+        help="Run an evaluation",
+    )
+    parser_eval.add_argument("file", help="A file containing the evaluation to run")
+
+    parsed = parser.parse_args()
+    if parsed.subcommand == "eval":
+        asyncio.run(run_evaluation(parsed))
+    else:
+        parser.print_help()

{lmnr-0.4.40 → lmnr-0.4.42}/src/lmnr/openllmetry_sdk/tracing/tracing.py

@@ -183,11 +183,7 @@ class TracerWrapper(object):
 
     @classmethod
     def verify_initialized(cls) -> bool:
-        if hasattr(cls, "instance"):
-            return True
-
-        cls.__logger.warning("Laminar not initialized, make sure to initialize")
-        return False
+        return hasattr(cls, "instance")
 
     def flush(self):
         self.__spans_processor.force_flush()

{lmnr-0.4.40 → lmnr-0.4.42}/src/lmnr/sdk/datasets.py

@@ -1,11 +1,9 @@
 from abc import ABC, abstractmethod
-import logging
+import asyncio
 
 from .log import get_default_logger
 from .laminar import Laminar as L
-from .types import (
-    Datapoint,
-)
+from .types import Datapoint
 
 DEFAULT_FETCH_SIZE = 25
 

lmnr-0.4.42/src/lmnr/sdk/eval_control.py

@@ -0,0 +1,4 @@
+from contextvars import ContextVar
+
+PREPARE_ONLY = ContextVar("__lmnr_prepare_only", default=False)
+EVALUATION_INSTANCE = ContextVar("__lmnr_evaluation_instance", default=None)

{lmnr-0.4.40 → lmnr-0.4.42}/src/lmnr/sdk/evaluations.py

@@ -3,7 +3,6 @@ import re
 import sys
 import uuid
 
-from contextlib import contextmanager
 from tqdm import tqdm
 from typing import Any, Awaitable, Optional, Set, Union
 
@@ -11,6 +10,7 @@ from ..openllmetry_sdk.instruments import Instruments
 from ..openllmetry_sdk.tracing.attributes import SPAN_TYPE
 
 from .datasets import EvaluationDataset
+from .eval_control import EVALUATION_INSTANCE, PREPARE_ONLY
 from .laminar import Laminar as L
 from .log import get_default_logger
 from .types import (
@@ -28,21 +28,6 @@ from .utils import is_async
 
 DEFAULT_BATCH_SIZE = 5
 
-_evaluation = None
-_set_global_evaluation = False
-
-
-@contextmanager
-def set_global_evaluation(set_global_evaluation: bool):
-    global _set_global_evaluation
-    original = _set_global_evaluation
-    try:
-        _set_global_evaluation = set_global_evaluation
-        yield
-    finally:
-        _set_global_evaluation = original
-        pass
-
 
 def get_evaluation_url(project_id: str, evaluation_id: str):
     return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
@@ -198,15 +183,10 @@ class Evaluation:
             instruments=instruments,
         )
 
-    def run(self) -> Union[None, Awaitable[None]]:
+    async def run(self) -> Awaitable[None]:
         if self.is_finished:
             raise Exception("Evaluation is already finished")
-
-        loop = asyncio.get_event_loop()
-        if loop.is_running():
-            return loop.create_task(self._run())
-        else:
-            return loop.run_until_complete(self._run())
+        return await self._run()
 
     async def _run(self) -> None:
         self.reporter.start(len(self.data))
@@ -224,7 +204,7 @@ class Evaluation:
         for result_datapoint in result_datapoints:
             result_datapoint.human_evaluators = self.human_evaluators or {}
 
-        evaluation = L.create_evaluation(
+        evaluation = await L.create_evaluation(
             data=result_datapoints, group_id=self.group_id, name=self.name
         )
         average_scores = get_average_scores(result_datapoints)
@@ -389,8 +369,11 @@ def evaluate(
         instruments=instruments,
     )
 
-    global _evaluation
-    if _set_global_evaluation:
-        _evaluation = evaluation
+    if PREPARE_ONLY.get():
+        EVALUATION_INSTANCE.set(evaluation)
     else:
-        return evaluation.run()
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            return loop.run_until_complete(evaluation.run())
+        else:
+            return asyncio.run(evaluation.run())

{lmnr-0.4.40 → lmnr-0.4.42}/src/lmnr/sdk/laminar.py

@@ -16,8 +16,10 @@ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExport
 from opentelemetry.util.types import AttributeValue
 
 from pydantic.alias_generators import to_snake
-from typing import Any, Literal, Optional, Set, Union
+from typing import Any, Awaitable, Literal, Optional, Set, Union
 
+import aiohttp
+import asyncio
 import copy
 import datetime
 import dotenv
@@ -25,8 +27,8 @@ import json
 import logging
 import os
 import random
-import re
 import requests
+import re
 import urllib.parse
 import uuid
 
@@ -54,6 +56,8 @@ from .types import (
     PipelineRunResponse,
     NodeInput,
     PipelineRunRequest,
+    SemanticSearchRequest,
+    SemanticSearchResponse,
     TraceType,
 )
 
@@ -64,7 +68,6 @@ class Laminar:
     __project_api_key: Optional[str] = None
     __env: dict[str, str] = {}
     __initialized: bool = False
-    __http_session: Optional[requests.Session] = None
 
     @classmethod
     def initialize(
@@ -129,7 +132,6 @@ class Laminar:
         cls.__env = env
         cls.__initialized = True
         cls._initialize_logger()
-        cls.__http_session = requests.Session()
         Traceloop.init(
             exporter=OTLPSpanExporter(
                 endpoint=cls.__base_grpc_url,
@@ -164,8 +166,9 @@ class Laminar:
         metadata: dict[str, str] = {},
         parent_span_id: Optional[uuid.UUID] = None,
         trace_id: Optional[uuid.UUID] = None,
-    ) -> PipelineRunResponse:
-        """Runs the pipeline with the given inputs
+    ) -> Union[PipelineRunResponse, Awaitable[PipelineRunResponse]]:
+        """Runs the pipeline with the given inputs. If called from an async
+        function, must be awaited.
 
         Args:
             pipeline (str): name of the Laminar pipeline.\
@@ -215,34 +218,47 @@ class Laminar:
                 parent_span_id=parent_span_id,
                 trace_id=trace_id,
             )
+            loop = asyncio.get_event_loop()
+            if loop.is_running():
+                return cls.__run(request)
+            else:
+                return asyncio.run(cls.__run(request))
         except Exception as e:
             raise ValueError(f"Invalid request: {e}")
 
-        response = (
-            cls.__http_session.post(
-                cls.__base_http_url + "/v1/pipeline/run",
-                data=json.dumps(request.to_dict()),
-                headers=cls._headers(),
-            )
-            if cls.__http_session
-            else requests.post(
-                cls.__base_http_url + "/v1/pipeline/run",
-                data=json.dumps(request.to_dict()),
-                headers=cls._headers(),
-            )
+    @classmethod
+    def semantic_search(
+        cls,
+        query: str,
+        dataset_id: uuid.UUID,
+        limit: Optional[int] = None,
+        threshold: Optional[float] = None,
+    ) -> SemanticSearchResponse:
+        """Perform a semantic search on a dataset. If called from an async
+        function, must be awaited.
+
+        Args:
+            query (str): query string to search by
+            dataset_id (uuid.UUID): id of the dataset to search in
+            limit (Optional[int], optional): maximum number of results to\
+                return. Defaults to None.
+            threshold (Optional[float], optional): minimum score for a result\
+                to be returned. Defaults to None.
+
+        Returns:
+            SemanticSearchResponse: response object containing the search results sorted by score in descending order
+        """
+        request = SemanticSearchRequest(
+            query=query,
+            dataset_id=dataset_id,
+            limit=limit,
+            threshold=threshold,
         )
-        if response.status_code != 200:
-            raise PipelineRunError(response)
-        try:
-            resp_json = response.json()
-            keys = list(resp_json.keys())
-            for key in keys:
-                value = resp_json[key]
-                del resp_json[key]
-                resp_json[to_snake(key)] = value
-            return PipelineRunResponse(**resp_json)
-        except Exception:
-            raise PipelineRunError(response)
+        loop = asyncio.get_event_loop()
+        if loop.is_running():
+            return cls.__semantic_search(request)
+        else:
+            return asyncio.run(cls.__semantic_search(request))
 
     @classmethod
     def event(
@@ -329,6 +345,9 @@ class Laminar:
                 span. Defaults to None.
         """
 
+        if not cls.is_initialized():
+            yield
+
         with get_tracer() as tracer:
             span_path = get_span_path(name)
             ctx = set_value("span_path", span_path, context)
@@ -646,30 +665,33 @@ class Laminar:
         set_association_properties(props)
 
     @classmethod
-    def create_evaluation(
+    async def create_evaluation(
         cls,
         data: list[EvaluationResultDatapoint],
         group_id: Optional[str] = None,
         name: Optional[str] = None,
     ) -> CreateEvaluationResponse:
-        response = requests.post(
-            cls.__base_http_url + "/v1/evaluations",
-            data=json.dumps(
-                {
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                cls.__base_http_url + "/v1/evaluations",
+                json={
                     "groupId": group_id,
                     "name": name,
                     "points": [datapoint.to_dict() for datapoint in data],
-                }
-            ),
-            headers=cls._headers(),
-        )
-        if response.status_code != 200:
-            try:
-                resp_json = response.json()
-                raise ValueError(f"Error creating evaluation {json.dumps(resp_json)}")
-            except requests.exceptions.RequestException:
-                raise ValueError(f"Error creating evaluation {response.text}")
-        return CreateEvaluationResponse.model_validate(response.json())
+                },
+                headers=cls._headers(),
+            ) as response:
+                if response.status != 200:
+                    try:
+                        resp_json = await response.json()
+                        raise ValueError(
+                            f"Error creating evaluation {json.dumps(resp_json)}"
+                        )
+                    except aiohttp.ClientError:
+                        text = await response.text()
+                        raise ValueError(f"Error creating evaluation {text}")
+                resp_json = await response.json()
+                return CreateEvaluationResponse.model_validate(resp_json)
 
     @classmethod
     def get_datapoints(
@@ -678,6 +700,10 @@ class Laminar:
         offset: int,
         limit: int,
     ) -> GetDatapointsResponse:
+        # TODO: Use aiohttp. Currently, this function is called from within
+        # `LaminarDataset.__len__`, which is sync, but can be called from
+        # both sync and async. Python does not make it easy to mix things this
+        # way, so we should probably refactor `LaminarDataset`.
         params = {"name": dataset_name, "offset": offset, "limit": limit}
         url = (
             cls.__base_http_url
@@ -704,3 +730,52 @@ class Laminar:
             "Authorization": "Bearer " + cls.__project_api_key,
             "Content-Type": "application/json",
         }
+
+    @classmethod
+    async def __run(
+        cls,
+        request: PipelineRunRequest,
+    ) -> PipelineRunResponse:
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                cls.__base_http_url + "/v1/pipeline/run",
+                data=json.dumps(request.to_dict()),
+                headers=cls._headers(),
+            ) as response:
+                if response.status != 200:
+                    raise PipelineRunError(response)
+                try:
+                    resp_json = await response.json()
+                    keys = list(resp_json.keys())
+                    for key in keys:
+                        value = resp_json[key]
+                        del resp_json[key]
+                        resp_json[to_snake(key)] = value
+                    return PipelineRunResponse(**resp_json)
+                except Exception:
+                    raise PipelineRunError(response)
+
+    @classmethod
+    async def __semantic_search(
+        cls,
+        request: SemanticSearchRequest,
+    ) -> SemanticSearchResponse:
+        async with aiohttp.ClientSession() as session:
+            async with session.post(
+                cls.__base_http_url + "/v1/semantic-search",
+                data=json.dumps(request.to_dict()),
+                headers=cls._headers(),
+            ) as response:
+                if response.status != 200:
+                    raise ValueError(
+                        f"Error performing semantic search: [{response.status}] {response.text}"
+                    )
+                try:
+                    resp_json = await response.json()
+                    for result in resp_json["results"]:
+                        result["dataset_id"] = uuid.UUID(result["datasetId"])
+                    return SemanticSearchResponse(**resp_json)
+                except Exception as e:
+                    raise ValueError(
+                        f"Error parsing semantic search response: status={response.status} error={e}"
+                    )

{lmnr-0.4.40 → lmnr-0.4.42}/src/lmnr/sdk/types.py

@@ -1,7 +1,7 @@
+import aiohttp
 import datetime
 from enum import Enum
 import pydantic
-import requests
 from typing import Any, Awaitable, Callable, Optional, Union
 import uuid
 
@@ -55,11 +55,40 @@ class PipelineRunResponse(pydantic.BaseModel):
     run_id: str
 
 
+class SemanticSearchRequest(pydantic.BaseModel):
+    query: str
+    dataset_id: uuid.UUID
+    limit: Optional[int] = pydantic.Field(default=None)
+    threshold: Optional[float] = pydantic.Field(default=None, ge=0.0, le=1.0)
+
+    def to_dict(self):
+        res = {
+            "query": self.query,
+            "datasetId": str(self.dataset_id),
+        }
+        if self.limit is not None:
+            res["limit"] = self.limit
+        if self.threshold is not None:
+            res["threshold"] = self.threshold
+        return res
+
+
+class SemanticSearchResult(pydantic.BaseModel):
+    dataset_id: uuid.UUID
+    score: float
+    data: dict[str, Any]
+    content: str
+
+
+class SemanticSearchResponse(pydantic.BaseModel):
+    results: list[SemanticSearchResult]
+
+
 class PipelineRunError(Exception):
     error_code: str
     error_message: str
 
-    def __init__(self, response: requests.Response):
+    def __init__(self, response: aiohttp.ClientResponse):
         try:
             resp_json = response.json()
             self.error_code = resp_json["error_code"]

lmnr-0.4.40/src/lmnr/cli.py

@@ -1,39 +0,0 @@
-from argparse import ArgumentParser
-import asyncio
-import importlib
-import os
-import sys
-
-from lmnr.sdk.evaluations import set_global_evaluation
-
-
-# TODO: Refactor this code
-async def run_evaluation(args):
-    sys.path.insert(0, os.getcwd())
-
-    with set_global_evaluation(True):
-        file = os.path.abspath(args.file)
-
-        spec = importlib.util.spec_from_file_location("run_eval", file)
-        mod = importlib.util.module_from_spec(spec)
-        spec.loader.exec_module(mod)
-
-        from lmnr.sdk.evaluations import _evaluation
-        evaluation = _evaluation
-        await evaluation.run()
-
-
-def cli():
-    parser = ArgumentParser(
-        prog="lmnr",
-        description="CLI for Laminar",
-    )
-
-    subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
-
-    parser_eval = subparsers.add_parser("eval", description="Run an evaluation")
-    parser_eval.add_argument("file", help="A file containing the evaluation to run")
-    parser_eval.set_defaults(func=run_evaluation)
-
-    parsed = parser.parse_args()
-    asyncio.run(parsed.func(parsed))