lmnr 0.4.39b1__py3-none-any.whl → 0.4.42__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lmnr/cli.py +25 -11
- lmnr/openllmetry_sdk/tracing/tracing.py +80 -33
- lmnr/openllmetry_sdk/utils/package_check.py +1 -0
- lmnr/sdk/datasets.py +2 -4
- lmnr/sdk/decorators.py +1 -9
- lmnr/sdk/eval_control.py +4 -0
- lmnr/sdk/evaluations.py +11 -28
- lmnr/sdk/laminar.py +123 -61
- lmnr/sdk/types.py +31 -2
- {lmnr-0.4.39b1.dist-info → lmnr-0.4.42.dist-info}/METADATA +62 -54
- {lmnr-0.4.39b1.dist-info → lmnr-0.4.42.dist-info}/RECORD +14 -13
- {lmnr-0.4.39b1.dist-info → lmnr-0.4.42.dist-info}/LICENSE +0 -0
- {lmnr-0.4.39b1.dist-info → lmnr-0.4.42.dist-info}/WHEEL +0 -0
- {lmnr-0.4.39b1.dist-info → lmnr-0.4.42.dist-info}/entry_points.txt +0 -0
lmnr/cli.py
CHANGED
@@ -1,26 +1,34 @@
|
|
1
1
|
from argparse import ArgumentParser
|
2
2
|
import asyncio
|
3
|
-
import importlib
|
3
|
+
import importlib.util
|
4
4
|
import os
|
5
5
|
import sys
|
6
6
|
|
7
|
-
from
|
7
|
+
from .sdk.eval_control import PREPARE_ONLY, EVALUATION_INSTANCE
|
8
8
|
|
9
9
|
|
10
|
-
# TODO: Refactor this code
|
11
10
|
async def run_evaluation(args):
|
12
|
-
sys.path.
|
11
|
+
sys.path.append(os.getcwd())
|
13
12
|
|
14
|
-
|
13
|
+
prep_token = PREPARE_ONLY.set(True)
|
14
|
+
try:
|
15
15
|
file = os.path.abspath(args.file)
|
16
|
+
name = "user_module"
|
16
17
|
|
17
|
-
spec = importlib.util.spec_from_file_location(
|
18
|
+
spec = importlib.util.spec_from_file_location(name, file)
|
19
|
+
if spec is None or spec.loader is None:
|
20
|
+
raise ImportError(f"Could not load module specification from {file}")
|
18
21
|
mod = importlib.util.module_from_spec(spec)
|
22
|
+
sys.modules[name] = mod
|
23
|
+
|
19
24
|
spec.loader.exec_module(mod)
|
25
|
+
evaluation = EVALUATION_INSTANCE.get()
|
26
|
+
if evaluation is None:
|
27
|
+
raise RuntimeError("Evaluation instance not found")
|
20
28
|
|
21
|
-
from lmnr.sdk.evaluations import _evaluation
|
22
|
-
evaluation = _evaluation
|
23
29
|
await evaluation.run()
|
30
|
+
finally:
|
31
|
+
PREPARE_ONLY.reset(prep_token)
|
24
32
|
|
25
33
|
|
26
34
|
def cli():
|
@@ -31,9 +39,15 @@ def cli():
|
|
31
39
|
|
32
40
|
subparsers = parser.add_subparsers(title="subcommands", dest="subcommand")
|
33
41
|
|
34
|
-
parser_eval = subparsers.add_parser(
|
42
|
+
parser_eval = subparsers.add_parser(
|
43
|
+
"eval",
|
44
|
+
description="Run an evaluation",
|
45
|
+
help="Run an evaluation",
|
46
|
+
)
|
35
47
|
parser_eval.add_argument("file", help="A file containing the evaluation to run")
|
36
|
-
parser_eval.set_defaults(func=run_evaluation)
|
37
48
|
|
38
49
|
parsed = parser.parse_args()
|
39
|
-
|
50
|
+
if parsed.subcommand == "eval":
|
51
|
+
asyncio.run(run_evaluation(parsed))
|
52
|
+
else:
|
53
|
+
parser.print_help()
|
@@ -183,11 +183,7 @@ class TracerWrapper(object):
|
|
183
183
|
|
184
184
|
@classmethod
|
185
185
|
def verify_initialized(cls) -> bool:
|
186
|
-
|
187
|
-
return True
|
188
|
-
|
189
|
-
cls.__logger.warning("Laminar not initialized, make sure to initialize")
|
190
|
-
return False
|
186
|
+
return hasattr(cls, "instance")
|
191
187
|
|
192
188
|
def flush(self):
|
193
189
|
self.__spans_processor.force_flush()
|
@@ -407,7 +403,9 @@ def init_instrumentations(
|
|
407
403
|
|
408
404
|
def init_openai_instrumentor(should_enrich_metrics: bool):
|
409
405
|
try:
|
410
|
-
if is_package_installed("openai")
|
406
|
+
if is_package_installed("openai") and is_package_installed(
|
407
|
+
"opentelemetry-instrumentation-openai"
|
408
|
+
):
|
411
409
|
from opentelemetry.instrumentation.openai import OpenAIInstrumentor
|
412
410
|
|
413
411
|
instrumentor = OpenAIInstrumentor(
|
@@ -425,7 +423,9 @@ def init_openai_instrumentor(should_enrich_metrics: bool):
|
|
425
423
|
|
426
424
|
def init_anthropic_instrumentor(should_enrich_metrics: bool):
|
427
425
|
try:
|
428
|
-
if is_package_installed("anthropic")
|
426
|
+
if is_package_installed("anthropic") and is_package_installed(
|
427
|
+
"opentelemetry-instrumentation-anthropic"
|
428
|
+
):
|
429
429
|
from opentelemetry.instrumentation.anthropic import AnthropicInstrumentor
|
430
430
|
|
431
431
|
instrumentor = AnthropicInstrumentor(
|
@@ -442,7 +442,9 @@ def init_anthropic_instrumentor(should_enrich_metrics: bool):
|
|
442
442
|
|
443
443
|
def init_cohere_instrumentor():
|
444
444
|
try:
|
445
|
-
if is_package_installed("cohere")
|
445
|
+
if is_package_installed("cohere") and is_package_installed(
|
446
|
+
"opentelemetry-instrumentation-cohere"
|
447
|
+
):
|
446
448
|
from opentelemetry.instrumentation.cohere import CohereInstrumentor
|
447
449
|
|
448
450
|
instrumentor = CohereInstrumentor()
|
@@ -456,7 +458,9 @@ def init_cohere_instrumentor():
|
|
456
458
|
|
457
459
|
def init_pinecone_instrumentor():
|
458
460
|
try:
|
459
|
-
if is_package_installed("pinecone")
|
461
|
+
if is_package_installed("pinecone") and is_package_installed(
|
462
|
+
"opentelemetry-instrumentation-pinecone"
|
463
|
+
):
|
460
464
|
from opentelemetry.instrumentation.pinecone import PineconeInstrumentor
|
461
465
|
|
462
466
|
instrumentor = PineconeInstrumentor()
|
@@ -470,7 +474,9 @@ def init_pinecone_instrumentor():
|
|
470
474
|
|
471
475
|
def init_qdrant_instrumentor():
|
472
476
|
try:
|
473
|
-
if is_package_installed("qdrant_client")
|
477
|
+
if is_package_installed("qdrant_client") and is_package_installed(
|
478
|
+
"opentelemetry-instrumentation-qdrant"
|
479
|
+
):
|
474
480
|
from opentelemetry.instrumentation.qdrant import QdrantInstrumentor
|
475
481
|
|
476
482
|
instrumentor = QdrantInstrumentor()
|
@@ -483,7 +489,9 @@ def init_qdrant_instrumentor():
|
|
483
489
|
|
484
490
|
def init_chroma_instrumentor():
|
485
491
|
try:
|
486
|
-
if is_package_installed("chromadb")
|
492
|
+
if is_package_installed("chromadb") and is_package_installed(
|
493
|
+
"opentelemetry-instrumentation-chromadb"
|
494
|
+
):
|
487
495
|
from opentelemetry.instrumentation.chromadb import ChromaInstrumentor
|
488
496
|
|
489
497
|
instrumentor = ChromaInstrumentor()
|
@@ -497,7 +505,9 @@ def init_chroma_instrumentor():
|
|
497
505
|
|
498
506
|
def init_google_generativeai_instrumentor():
|
499
507
|
try:
|
500
|
-
if is_package_installed("google.generativeai")
|
508
|
+
if is_package_installed("google.generativeai") and is_package_installed(
|
509
|
+
"opentelemetry-instrumentation-google-generativeai"
|
510
|
+
):
|
501
511
|
from opentelemetry.instrumentation.google_generativeai import (
|
502
512
|
GoogleGenerativeAiInstrumentor,
|
503
513
|
)
|
@@ -513,7 +523,9 @@ def init_google_generativeai_instrumentor():
|
|
513
523
|
|
514
524
|
def init_haystack_instrumentor():
|
515
525
|
try:
|
516
|
-
if is_package_installed("haystack")
|
526
|
+
if is_package_installed("haystack") and is_package_installed(
|
527
|
+
"opentelemetry-instrumentation-haystack"
|
528
|
+
):
|
517
529
|
from opentelemetry.instrumentation.haystack import HaystackInstrumentor
|
518
530
|
|
519
531
|
instrumentor = HaystackInstrumentor()
|
@@ -527,7 +539,9 @@ def init_haystack_instrumentor():
|
|
527
539
|
|
528
540
|
def init_langchain_instrumentor():
|
529
541
|
try:
|
530
|
-
if is_package_installed("langchain")
|
542
|
+
if is_package_installed("langchain") and is_package_installed(
|
543
|
+
"opentelemetry-instrumentation-langchain"
|
544
|
+
):
|
531
545
|
from opentelemetry.instrumentation.langchain import LangchainInstrumentor
|
532
546
|
|
533
547
|
instrumentor = LangchainInstrumentor()
|
@@ -543,7 +557,9 @@ def init_langchain_instrumentor():
|
|
543
557
|
|
544
558
|
def init_mistralai_instrumentor():
|
545
559
|
try:
|
546
|
-
if is_package_installed("mistralai")
|
560
|
+
if is_package_installed("mistralai") and is_package_installed(
|
561
|
+
"opentelemetry-instrumentation-mistralai"
|
562
|
+
):
|
547
563
|
from opentelemetry.instrumentation.mistralai import MistralAiInstrumentor
|
548
564
|
|
549
565
|
instrumentor = MistralAiInstrumentor()
|
@@ -557,7 +573,9 @@ def init_mistralai_instrumentor():
|
|
557
573
|
|
558
574
|
def init_ollama_instrumentor():
|
559
575
|
try:
|
560
|
-
if is_package_installed("ollama")
|
576
|
+
if is_package_installed("ollama") and is_package_installed(
|
577
|
+
"opentelemetry-instrumentation-ollama"
|
578
|
+
):
|
561
579
|
from opentelemetry.instrumentation.ollama import OllamaInstrumentor
|
562
580
|
|
563
581
|
instrumentor = OllamaInstrumentor()
|
@@ -571,7 +589,9 @@ def init_ollama_instrumentor():
|
|
571
589
|
|
572
590
|
def init_transformers_instrumentor():
|
573
591
|
try:
|
574
|
-
if is_package_installed("transformers")
|
592
|
+
if is_package_installed("transformers") and is_package_installed(
|
593
|
+
"opentelemetry-instrumentation-transformers"
|
594
|
+
):
|
575
595
|
from opentelemetry.instrumentation.transformers import (
|
576
596
|
TransformersInstrumentor,
|
577
597
|
)
|
@@ -587,7 +607,9 @@ def init_transformers_instrumentor():
|
|
587
607
|
|
588
608
|
def init_together_instrumentor():
|
589
609
|
try:
|
590
|
-
if is_package_installed("together")
|
610
|
+
if is_package_installed("together") and is_package_installed(
|
611
|
+
"opentelemetry-instrumentation-together"
|
612
|
+
):
|
591
613
|
from opentelemetry.instrumentation.together import TogetherAiInstrumentor
|
592
614
|
|
593
615
|
instrumentor = TogetherAiInstrumentor()
|
@@ -601,7 +623,9 @@ def init_together_instrumentor():
|
|
601
623
|
|
602
624
|
def init_llama_index_instrumentor():
|
603
625
|
try:
|
604
|
-
if
|
626
|
+
if (
|
627
|
+
is_package_installed("llama-index") or is_package_installed("llama_index")
|
628
|
+
) and is_package_installed("opentelemetry-instrumentation-llamaindex"):
|
605
629
|
from opentelemetry.instrumentation.llamaindex import LlamaIndexInstrumentor
|
606
630
|
|
607
631
|
instrumentor = LlamaIndexInstrumentor()
|
@@ -615,7 +639,9 @@ def init_llama_index_instrumentor():
|
|
615
639
|
|
616
640
|
def init_milvus_instrumentor():
|
617
641
|
try:
|
618
|
-
if is_package_installed("pymilvus")
|
642
|
+
if is_package_installed("pymilvus") and is_package_installed(
|
643
|
+
"opentelemetry-instrumentation-milvus"
|
644
|
+
):
|
619
645
|
from opentelemetry.instrumentation.milvus import MilvusInstrumentor
|
620
646
|
|
621
647
|
instrumentor = MilvusInstrumentor()
|
@@ -671,7 +697,9 @@ def init_pymysql_instrumentor():
|
|
671
697
|
|
672
698
|
def init_bedrock_instrumentor(should_enrich_metrics: bool):
|
673
699
|
try:
|
674
|
-
if is_package_installed("boto3")
|
700
|
+
if is_package_installed("boto3") and is_package_installed(
|
701
|
+
"opentelemetry-instrumentation-bedrock"
|
702
|
+
):
|
675
703
|
from opentelemetry.instrumentation.bedrock import BedrockInstrumentor
|
676
704
|
|
677
705
|
instrumentor = BedrockInstrumentor(
|
@@ -687,7 +715,9 @@ def init_bedrock_instrumentor(should_enrich_metrics: bool):
|
|
687
715
|
|
688
716
|
def init_replicate_instrumentor():
|
689
717
|
try:
|
690
|
-
if is_package_installed("replicate")
|
718
|
+
if is_package_installed("replicate") and is_package_installed(
|
719
|
+
"opentelemetry-instrumentation-replicate"
|
720
|
+
):
|
691
721
|
from opentelemetry.instrumentation.replicate import ReplicateInstrumentor
|
692
722
|
|
693
723
|
instrumentor = ReplicateInstrumentor()
|
@@ -701,7 +731,9 @@ def init_replicate_instrumentor():
|
|
701
731
|
|
702
732
|
def init_vertexai_instrumentor():
|
703
733
|
try:
|
704
|
-
if is_package_installed("vertexai")
|
734
|
+
if is_package_installed("vertexai") and is_package_installed(
|
735
|
+
"opentelemetry-instrumentation-vertexai"
|
736
|
+
):
|
705
737
|
from opentelemetry.instrumentation.vertexai import VertexAIInstrumentor
|
706
738
|
|
707
739
|
instrumentor = VertexAIInstrumentor()
|
@@ -715,9 +747,10 @@ def init_vertexai_instrumentor():
|
|
715
747
|
|
716
748
|
def init_watsonx_instrumentor():
|
717
749
|
try:
|
718
|
-
if
|
719
|
-
"ibm-
|
720
|
-
|
750
|
+
if (
|
751
|
+
is_package_installed("ibm-watsonx-ai")
|
752
|
+
or is_package_installed("ibm-watson-machine-learning")
|
753
|
+
) and is_package_installed("opentelemetry-instrumentation-watsonx"):
|
721
754
|
from opentelemetry.instrumentation.watsonx import WatsonxInstrumentor
|
722
755
|
|
723
756
|
instrumentor = WatsonxInstrumentor()
|
@@ -731,7 +764,9 @@ def init_watsonx_instrumentor():
|
|
731
764
|
|
732
765
|
def init_weaviate_instrumentor():
|
733
766
|
try:
|
734
|
-
if is_package_installed("weaviate")
|
767
|
+
if is_package_installed("weaviate") and is_package_installed(
|
768
|
+
"opentelemetry-instrumentation-weaviate"
|
769
|
+
):
|
735
770
|
from opentelemetry.instrumentation.weaviate import WeaviateInstrumentor
|
736
771
|
|
737
772
|
instrumentor = WeaviateInstrumentor()
|
@@ -745,7 +780,9 @@ def init_weaviate_instrumentor():
|
|
745
780
|
|
746
781
|
def init_alephalpha_instrumentor():
|
747
782
|
try:
|
748
|
-
if is_package_installed("aleph_alpha_client")
|
783
|
+
if is_package_installed("aleph_alpha_client") and is_package_installed(
|
784
|
+
"opentelemetry-instrumentation-alephalpha"
|
785
|
+
):
|
749
786
|
from opentelemetry.instrumentation.alephalpha import AlephAlphaInstrumentor
|
750
787
|
|
751
788
|
instrumentor = AlephAlphaInstrumentor()
|
@@ -759,7 +796,9 @@ def init_alephalpha_instrumentor():
|
|
759
796
|
|
760
797
|
def init_marqo_instrumentor():
|
761
798
|
try:
|
762
|
-
if is_package_installed("marqo")
|
799
|
+
if is_package_installed("marqo") and is_package_installed(
|
800
|
+
"opentelemetry-instrumentation-marqo"
|
801
|
+
):
|
763
802
|
from opentelemetry.instrumentation.marqo import MarqoInstrumentor
|
764
803
|
|
765
804
|
instrumentor = MarqoInstrumentor()
|
@@ -773,7 +812,9 @@ def init_marqo_instrumentor():
|
|
773
812
|
|
774
813
|
def init_lancedb_instrumentor():
|
775
814
|
try:
|
776
|
-
if is_package_installed("lancedb")
|
815
|
+
if is_package_installed("lancedb") and is_package_installed(
|
816
|
+
"opentelemetry-instrumentation-lancedb"
|
817
|
+
):
|
777
818
|
from opentelemetry.instrumentation.lancedb import LanceInstrumentor
|
778
819
|
|
779
820
|
instrumentor = LanceInstrumentor()
|
@@ -786,7 +827,9 @@ def init_lancedb_instrumentor():
|
|
786
827
|
|
787
828
|
def init_redis_instrumentor():
|
788
829
|
try:
|
789
|
-
if is_package_installed("redis")
|
830
|
+
if is_package_installed("redis") and is_package_installed(
|
831
|
+
"opentelemetry-instrumentation-redis"
|
832
|
+
):
|
790
833
|
from opentelemetry.instrumentation.redis import RedisInstrumentor
|
791
834
|
|
792
835
|
instrumentor = RedisInstrumentor()
|
@@ -800,7 +843,9 @@ def init_redis_instrumentor():
|
|
800
843
|
|
801
844
|
def init_groq_instrumentor():
|
802
845
|
try:
|
803
|
-
if is_package_installed("groq")
|
846
|
+
if is_package_installed("groq") and is_package_installed(
|
847
|
+
"opentelemetry-instrumentation-groq"
|
848
|
+
):
|
804
849
|
from opentelemetry.instrumentation.groq import GroqInstrumentor
|
805
850
|
|
806
851
|
instrumentor = GroqInstrumentor()
|
@@ -814,7 +859,9 @@ def init_groq_instrumentor():
|
|
814
859
|
|
815
860
|
def init_sagemaker_instrumentor(should_enrich_metrics: bool):
|
816
861
|
try:
|
817
|
-
if is_package_installed("boto3")
|
862
|
+
if is_package_installed("boto3") and is_package_installed(
|
863
|
+
"opentelemetry-instrumentation-sagemaker"
|
864
|
+
):
|
818
865
|
from opentelemetry.instrumentation.sagemaker import SageMakerInstrumentor
|
819
866
|
|
820
867
|
instrumentor = SageMakerInstrumentor(
|
lmnr/sdk/datasets.py
CHANGED
lmnr/sdk/decorators.py
CHANGED
@@ -7,7 +7,7 @@ from opentelemetry.trace import INVALID_SPAN, get_current_span
|
|
7
7
|
from typing import Callable, Optional, TypeVar, cast
|
8
8
|
from typing_extensions import ParamSpec
|
9
9
|
|
10
|
-
from lmnr.openllmetry_sdk.tracing.attributes import SESSION_ID
|
10
|
+
from lmnr.openllmetry_sdk.tracing.attributes import SESSION_ID
|
11
11
|
from lmnr.openllmetry_sdk.tracing.tracing import update_association_properties
|
12
12
|
|
13
13
|
from .utils import is_async
|
@@ -20,7 +20,6 @@ R = TypeVar("R")
|
|
20
20
|
def observe(
|
21
21
|
*,
|
22
22
|
name: Optional[str] = None,
|
23
|
-
user_id: Optional[str] = None,
|
24
23
|
session_id: Optional[str] = None,
|
25
24
|
) -> Callable[[Callable[P, R]], Callable[P, R]]:
|
26
25
|
"""The main decorator entrypoint for Laminar. This is used to wrap
|
@@ -30,9 +29,6 @@ def observe(
|
|
30
29
|
name (Optional[str], optional): Name of the span. Function
|
31
30
|
name is used if not specified.
|
32
31
|
Defaults to None.
|
33
|
-
user_id (Optional[str], optional): User ID to associate
|
34
|
-
with the span and the following context.
|
35
|
-
Defaults to None.
|
36
32
|
session_id (Optional[str], optional): Session ID to associate with the
|
37
33
|
span and the following context. Defaults to None.
|
38
34
|
|
@@ -49,13 +45,9 @@ def observe(
|
|
49
45
|
if current_span != INVALID_SPAN:
|
50
46
|
if session_id is not None:
|
51
47
|
current_span.set_attribute(SESSION_ID, session_id)
|
52
|
-
if user_id is not None:
|
53
|
-
current_span.set_attribute(USER_ID, user_id)
|
54
48
|
association_properties = {}
|
55
49
|
if session_id is not None:
|
56
50
|
association_properties["session_id"] = session_id
|
57
|
-
if user_id is not None:
|
58
|
-
association_properties["user_id"] = user_id
|
59
51
|
update_association_properties(association_properties)
|
60
52
|
return (
|
61
53
|
aentity_method(name=name)(func)
|
lmnr/sdk/eval_control.py
ADDED
lmnr/sdk/evaluations.py
CHANGED
@@ -3,7 +3,6 @@ import re
|
|
3
3
|
import sys
|
4
4
|
import uuid
|
5
5
|
|
6
|
-
from contextlib import contextmanager
|
7
6
|
from tqdm import tqdm
|
8
7
|
from typing import Any, Awaitable, Optional, Set, Union
|
9
8
|
|
@@ -11,6 +10,7 @@ from ..openllmetry_sdk.instruments import Instruments
|
|
11
10
|
from ..openllmetry_sdk.tracing.attributes import SPAN_TYPE
|
12
11
|
|
13
12
|
from .datasets import EvaluationDataset
|
13
|
+
from .eval_control import EVALUATION_INSTANCE, PREPARE_ONLY
|
14
14
|
from .laminar import Laminar as L
|
15
15
|
from .log import get_default_logger
|
16
16
|
from .types import (
|
@@ -28,21 +28,6 @@ from .utils import is_async
|
|
28
28
|
|
29
29
|
DEFAULT_BATCH_SIZE = 5
|
30
30
|
|
31
|
-
_evaluation = None
|
32
|
-
_set_global_evaluation = False
|
33
|
-
|
34
|
-
|
35
|
-
@contextmanager
|
36
|
-
def set_global_evaluation(set_global_evaluation: bool):
|
37
|
-
global _set_global_evaluation
|
38
|
-
original = _set_global_evaluation
|
39
|
-
try:
|
40
|
-
_set_global_evaluation = set_global_evaluation
|
41
|
-
yield
|
42
|
-
finally:
|
43
|
-
_set_global_evaluation = original
|
44
|
-
pass
|
45
|
-
|
46
31
|
|
47
32
|
def get_evaluation_url(project_id: str, evaluation_id: str):
|
48
33
|
return f"https://www.lmnr.ai/project/{project_id}/evaluations/{evaluation_id}"
|
@@ -198,15 +183,10 @@ class Evaluation:
|
|
198
183
|
instruments=instruments,
|
199
184
|
)
|
200
185
|
|
201
|
-
def run(self) ->
|
186
|
+
async def run(self) -> Awaitable[None]:
|
202
187
|
if self.is_finished:
|
203
188
|
raise Exception("Evaluation is already finished")
|
204
|
-
|
205
|
-
loop = asyncio.get_event_loop()
|
206
|
-
if loop.is_running():
|
207
|
-
return loop.create_task(self._run())
|
208
|
-
else:
|
209
|
-
return loop.run_until_complete(self._run())
|
189
|
+
return await self._run()
|
210
190
|
|
211
191
|
async def _run(self) -> None:
|
212
192
|
self.reporter.start(len(self.data))
|
@@ -224,7 +204,7 @@ class Evaluation:
|
|
224
204
|
for result_datapoint in result_datapoints:
|
225
205
|
result_datapoint.human_evaluators = self.human_evaluators or {}
|
226
206
|
|
227
|
-
evaluation = L.create_evaluation(
|
207
|
+
evaluation = await L.create_evaluation(
|
228
208
|
data=result_datapoints, group_id=self.group_id, name=self.name
|
229
209
|
)
|
230
210
|
average_scores = get_average_scores(result_datapoints)
|
@@ -389,8 +369,11 @@ def evaluate(
|
|
389
369
|
instruments=instruments,
|
390
370
|
)
|
391
371
|
|
392
|
-
|
393
|
-
|
394
|
-
_evaluation = evaluation
|
372
|
+
if PREPARE_ONLY.get():
|
373
|
+
EVALUATION_INSTANCE.set(evaluation)
|
395
374
|
else:
|
396
|
-
|
375
|
+
loop = asyncio.get_event_loop()
|
376
|
+
if loop.is_running():
|
377
|
+
return loop.run_until_complete(evaluation.run())
|
378
|
+
else:
|
379
|
+
return asyncio.run(evaluation.run())
|
lmnr/sdk/laminar.py
CHANGED
@@ -16,8 +16,10 @@ from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExport
|
|
16
16
|
from opentelemetry.util.types import AttributeValue
|
17
17
|
|
18
18
|
from pydantic.alias_generators import to_snake
|
19
|
-
from typing import Any, Literal, Optional, Set, Union
|
19
|
+
from typing import Any, Awaitable, Literal, Optional, Set, Union
|
20
20
|
|
21
|
+
import aiohttp
|
22
|
+
import asyncio
|
21
23
|
import copy
|
22
24
|
import datetime
|
23
25
|
import dotenv
|
@@ -25,8 +27,8 @@ import json
|
|
25
27
|
import logging
|
26
28
|
import os
|
27
29
|
import random
|
28
|
-
import re
|
29
30
|
import requests
|
31
|
+
import re
|
30
32
|
import urllib.parse
|
31
33
|
import uuid
|
32
34
|
|
@@ -36,7 +38,6 @@ from lmnr.openllmetry_sdk.tracing.attributes import (
|
|
36
38
|
SPAN_OUTPUT,
|
37
39
|
SPAN_PATH,
|
38
40
|
TRACE_TYPE,
|
39
|
-
USER_ID,
|
40
41
|
)
|
41
42
|
from lmnr.openllmetry_sdk.tracing.tracing import (
|
42
43
|
get_span_path,
|
@@ -55,6 +56,8 @@ from .types import (
|
|
55
56
|
PipelineRunResponse,
|
56
57
|
NodeInput,
|
57
58
|
PipelineRunRequest,
|
59
|
+
SemanticSearchRequest,
|
60
|
+
SemanticSearchResponse,
|
58
61
|
TraceType,
|
59
62
|
)
|
60
63
|
|
@@ -65,7 +68,6 @@ class Laminar:
|
|
65
68
|
__project_api_key: Optional[str] = None
|
66
69
|
__env: dict[str, str] = {}
|
67
70
|
__initialized: bool = False
|
68
|
-
__http_session: Optional[requests.Session] = None
|
69
71
|
|
70
72
|
@classmethod
|
71
73
|
def initialize(
|
@@ -130,7 +132,6 @@ class Laminar:
|
|
130
132
|
cls.__env = env
|
131
133
|
cls.__initialized = True
|
132
134
|
cls._initialize_logger()
|
133
|
-
cls.__http_session = requests.Session()
|
134
135
|
Traceloop.init(
|
135
136
|
exporter=OTLPSpanExporter(
|
136
137
|
endpoint=cls.__base_grpc_url,
|
@@ -165,8 +166,9 @@ class Laminar:
|
|
165
166
|
metadata: dict[str, str] = {},
|
166
167
|
parent_span_id: Optional[uuid.UUID] = None,
|
167
168
|
trace_id: Optional[uuid.UUID] = None,
|
168
|
-
) -> PipelineRunResponse:
|
169
|
-
"""Runs the pipeline with the given inputs
|
169
|
+
) -> Union[PipelineRunResponse, Awaitable[PipelineRunResponse]]:
|
170
|
+
"""Runs the pipeline with the given inputs. If called from an async
|
171
|
+
function, must be awaited.
|
170
172
|
|
171
173
|
Args:
|
172
174
|
pipeline (str): name of the Laminar pipeline.\
|
@@ -216,34 +218,47 @@ class Laminar:
|
|
216
218
|
parent_span_id=parent_span_id,
|
217
219
|
trace_id=trace_id,
|
218
220
|
)
|
221
|
+
loop = asyncio.get_event_loop()
|
222
|
+
if loop.is_running():
|
223
|
+
return cls.__run(request)
|
224
|
+
else:
|
225
|
+
return asyncio.run(cls.__run(request))
|
219
226
|
except Exception as e:
|
220
227
|
raise ValueError(f"Invalid request: {e}")
|
221
228
|
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
229
|
+
@classmethod
|
230
|
+
def semantic_search(
|
231
|
+
cls,
|
232
|
+
query: str,
|
233
|
+
dataset_id: uuid.UUID,
|
234
|
+
limit: Optional[int] = None,
|
235
|
+
threshold: Optional[float] = None,
|
236
|
+
) -> SemanticSearchResponse:
|
237
|
+
"""Perform a semantic search on a dataset. If called from an async
|
238
|
+
function, must be awaited.
|
239
|
+
|
240
|
+
Args:
|
241
|
+
query (str): query string to search by
|
242
|
+
dataset_id (uuid.UUID): id of the dataset to search in
|
243
|
+
limit (Optional[int], optional): maximum number of results to\
|
244
|
+
return. Defaults to None.
|
245
|
+
threshold (Optional[float], optional): minimum score for a result\
|
246
|
+
to be returned. Defaults to None.
|
247
|
+
|
248
|
+
Returns:
|
249
|
+
SemanticSearchResponse: response object containing the search results sorted by score in descending order
|
250
|
+
"""
|
251
|
+
request = SemanticSearchRequest(
|
252
|
+
query=query,
|
253
|
+
dataset_id=dataset_id,
|
254
|
+
limit=limit,
|
255
|
+
threshold=threshold,
|
234
256
|
)
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
for key in keys:
|
241
|
-
value = resp_json[key]
|
242
|
-
del resp_json[key]
|
243
|
-
resp_json[to_snake(key)] = value
|
244
|
-
return PipelineRunResponse(**resp_json)
|
245
|
-
except Exception:
|
246
|
-
raise PipelineRunError(response)
|
257
|
+
loop = asyncio.get_event_loop()
|
258
|
+
if loop.is_running():
|
259
|
+
return cls.__semantic_search(request)
|
260
|
+
else:
|
261
|
+
return asyncio.run(cls.__semantic_search(request))
|
247
262
|
|
248
263
|
@classmethod
|
249
264
|
def event(
|
@@ -330,6 +345,9 @@ class Laminar:
|
|
330
345
|
span. Defaults to None.
|
331
346
|
"""
|
332
347
|
|
348
|
+
if not cls.is_initialized():
|
349
|
+
yield
|
350
|
+
|
333
351
|
with get_tracer() as tracer:
|
334
352
|
span_path = get_span_path(name)
|
335
353
|
ctx = set_value("span_path", span_path, context)
|
@@ -588,7 +606,6 @@ class Laminar:
|
|
588
606
|
def set_session(
|
589
607
|
cls,
|
590
608
|
session_id: Optional[str] = None,
|
591
|
-
user_id: Optional[str] = None,
|
592
609
|
):
|
593
610
|
"""Set the session and user id for the current span and the context
|
594
611
|
(i.e. any children spans created from the current span in the current
|
@@ -599,29 +616,18 @@ class Laminar:
|
|
599
616
|
Useful to debug and group long-running\
|
600
617
|
sessions/conversations.
|
601
618
|
Defaults to None.
|
602
|
-
user_id (Optional[str], optional). Deprecated.\
|
603
|
-
Use `Laminar.set_metadata` instead.\
|
604
|
-
Custom user id.\
|
605
|
-
Useful for grouping spans or traces by user.\
|
606
|
-
Defaults to None.
|
607
619
|
"""
|
608
620
|
association_properties = {}
|
609
621
|
if session_id is not None:
|
610
622
|
association_properties[SESSION_ID] = session_id
|
611
|
-
if user_id is not None:
|
612
|
-
cls.__logger.warning(
|
613
|
-
"User ID in set_session is deprecated and will be removed soon. "
|
614
|
-
"Please use `Laminar.set_metadata` instead."
|
615
|
-
)
|
616
|
-
association_properties["metadata." + USER_ID] = user_id
|
617
623
|
update_association_properties(association_properties)
|
618
624
|
|
619
625
|
@classmethod
|
620
|
-
def set_metadata(cls, metadata: dict[str,
|
626
|
+
def set_metadata(cls, metadata: dict[str, str]):
|
621
627
|
"""Set the metadata for the current trace.
|
622
628
|
|
623
629
|
Args:
|
624
|
-
metadata (dict[str,
|
630
|
+
metadata (dict[str, str]): Metadata to set for the trace. Willl be\
|
625
631
|
sent as attributes, so must be json serializable.
|
626
632
|
"""
|
627
633
|
props = {f"metadata.{k}": json_dumps(v) for k, v in metadata.items()}
|
@@ -659,30 +665,33 @@ class Laminar:
|
|
659
665
|
set_association_properties(props)
|
660
666
|
|
661
667
|
@classmethod
|
662
|
-
def create_evaluation(
|
668
|
+
async def create_evaluation(
|
663
669
|
cls,
|
664
670
|
data: list[EvaluationResultDatapoint],
|
665
671
|
group_id: Optional[str] = None,
|
666
672
|
name: Optional[str] = None,
|
667
673
|
) -> CreateEvaluationResponse:
|
668
|
-
|
669
|
-
|
670
|
-
|
671
|
-
{
|
674
|
+
async with aiohttp.ClientSession() as session:
|
675
|
+
async with session.post(
|
676
|
+
cls.__base_http_url + "/v1/evaluations",
|
677
|
+
json={
|
672
678
|
"groupId": group_id,
|
673
679
|
"name": name,
|
674
680
|
"points": [datapoint.to_dict() for datapoint in data],
|
675
|
-
}
|
676
|
-
|
677
|
-
|
678
|
-
|
679
|
-
|
680
|
-
|
681
|
-
|
682
|
-
|
683
|
-
|
684
|
-
|
685
|
-
|
681
|
+
},
|
682
|
+
headers=cls._headers(),
|
683
|
+
) as response:
|
684
|
+
if response.status != 200:
|
685
|
+
try:
|
686
|
+
resp_json = await response.json()
|
687
|
+
raise ValueError(
|
688
|
+
f"Error creating evaluation {json.dumps(resp_json)}"
|
689
|
+
)
|
690
|
+
except aiohttp.ClientError:
|
691
|
+
text = await response.text()
|
692
|
+
raise ValueError(f"Error creating evaluation {text}")
|
693
|
+
resp_json = await response.json()
|
694
|
+
return CreateEvaluationResponse.model_validate(resp_json)
|
686
695
|
|
687
696
|
@classmethod
|
688
697
|
def get_datapoints(
|
@@ -691,6 +700,10 @@ class Laminar:
|
|
691
700
|
offset: int,
|
692
701
|
limit: int,
|
693
702
|
) -> GetDatapointsResponse:
|
703
|
+
# TODO: Use aiohttp. Currently, this function is called from within
|
704
|
+
# `LaminarDataset.__len__`, which is sync, but can be called from
|
705
|
+
# both sync and async. Python does not make it easy to mix things this
|
706
|
+
# way, so we should probably refactor `LaminarDataset`.
|
694
707
|
params = {"name": dataset_name, "offset": offset, "limit": limit}
|
695
708
|
url = (
|
696
709
|
cls.__base_http_url
|
@@ -717,3 +730,52 @@ class Laminar:
|
|
717
730
|
"Authorization": "Bearer " + cls.__project_api_key,
|
718
731
|
"Content-Type": "application/json",
|
719
732
|
}
|
733
|
+
|
734
|
+
@classmethod
|
735
|
+
async def __run(
|
736
|
+
cls,
|
737
|
+
request: PipelineRunRequest,
|
738
|
+
) -> PipelineRunResponse:
|
739
|
+
async with aiohttp.ClientSession() as session:
|
740
|
+
async with session.post(
|
741
|
+
cls.__base_http_url + "/v1/pipeline/run",
|
742
|
+
data=json.dumps(request.to_dict()),
|
743
|
+
headers=cls._headers(),
|
744
|
+
) as response:
|
745
|
+
if response.status != 200:
|
746
|
+
raise PipelineRunError(response)
|
747
|
+
try:
|
748
|
+
resp_json = await response.json()
|
749
|
+
keys = list(resp_json.keys())
|
750
|
+
for key in keys:
|
751
|
+
value = resp_json[key]
|
752
|
+
del resp_json[key]
|
753
|
+
resp_json[to_snake(key)] = value
|
754
|
+
return PipelineRunResponse(**resp_json)
|
755
|
+
except Exception:
|
756
|
+
raise PipelineRunError(response)
|
757
|
+
|
758
|
+
@classmethod
|
759
|
+
async def __semantic_search(
|
760
|
+
cls,
|
761
|
+
request: SemanticSearchRequest,
|
762
|
+
) -> SemanticSearchResponse:
|
763
|
+
async with aiohttp.ClientSession() as session:
|
764
|
+
async with session.post(
|
765
|
+
cls.__base_http_url + "/v1/semantic-search",
|
766
|
+
data=json.dumps(request.to_dict()),
|
767
|
+
headers=cls._headers(),
|
768
|
+
) as response:
|
769
|
+
if response.status != 200:
|
770
|
+
raise ValueError(
|
771
|
+
f"Error performing semantic search: [{response.status}] {response.text}"
|
772
|
+
)
|
773
|
+
try:
|
774
|
+
resp_json = await response.json()
|
775
|
+
for result in resp_json["results"]:
|
776
|
+
result["dataset_id"] = uuid.UUID(result["datasetId"])
|
777
|
+
return SemanticSearchResponse(**resp_json)
|
778
|
+
except Exception as e:
|
779
|
+
raise ValueError(
|
780
|
+
f"Error parsing semantic search response: status={response.status} error={e}"
|
781
|
+
)
|
lmnr/sdk/types.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1
|
+
import aiohttp
|
1
2
|
import datetime
|
2
3
|
from enum import Enum
|
3
4
|
import pydantic
|
4
|
-
import requests
|
5
5
|
from typing import Any, Awaitable, Callable, Optional, Union
|
6
6
|
import uuid
|
7
7
|
|
@@ -55,11 +55,40 @@ class PipelineRunResponse(pydantic.BaseModel):
|
|
55
55
|
run_id: str
|
56
56
|
|
57
57
|
|
58
|
+
class SemanticSearchRequest(pydantic.BaseModel):
|
59
|
+
query: str
|
60
|
+
dataset_id: uuid.UUID
|
61
|
+
limit: Optional[int] = pydantic.Field(default=None)
|
62
|
+
threshold: Optional[float] = pydantic.Field(default=None, ge=0.0, le=1.0)
|
63
|
+
|
64
|
+
def to_dict(self):
|
65
|
+
res = {
|
66
|
+
"query": self.query,
|
67
|
+
"datasetId": str(self.dataset_id),
|
68
|
+
}
|
69
|
+
if self.limit is not None:
|
70
|
+
res["limit"] = self.limit
|
71
|
+
if self.threshold is not None:
|
72
|
+
res["threshold"] = self.threshold
|
73
|
+
return res
|
74
|
+
|
75
|
+
|
76
|
+
class SemanticSearchResult(pydantic.BaseModel):
|
77
|
+
dataset_id: uuid.UUID
|
78
|
+
score: float
|
79
|
+
data: dict[str, Any]
|
80
|
+
content: str
|
81
|
+
|
82
|
+
|
83
|
+
class SemanticSearchResponse(pydantic.BaseModel):
|
84
|
+
results: list[SemanticSearchResult]
|
85
|
+
|
86
|
+
|
58
87
|
class PipelineRunError(Exception):
|
59
88
|
error_code: str
|
60
89
|
error_message: str
|
61
90
|
|
62
|
-
def __init__(self, response:
|
91
|
+
def __init__(self, response: aiohttp.ClientResponse):
|
63
92
|
try:
|
64
93
|
resp_json = response.json()
|
65
94
|
self.error_code = resp_json["error_code"]
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: lmnr
|
3
|
-
Version: 0.4.
|
3
|
+
Version: 0.4.42
|
4
4
|
Summary: Python SDK for Laminar AI
|
5
5
|
License: Apache-2.0
|
6
6
|
Author: lmnr.ai
|
@@ -13,11 +13,11 @@ Classifier: Programming Language :: Python :: 3.11
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
14
14
|
Classifier: Programming Language :: Python :: 3.13
|
15
15
|
Provides-Extra: alephalpha
|
16
|
+
Provides-Extra: all
|
16
17
|
Provides-Extra: anthropic
|
17
18
|
Provides-Extra: bedrock
|
18
19
|
Provides-Extra: chromadb
|
19
20
|
Provides-Extra: cohere
|
20
|
-
Provides-Extra: full
|
21
21
|
Provides-Extra: google-generativeai
|
22
22
|
Provides-Extra: groq
|
23
23
|
Provides-Extra: haystack
|
@@ -38,6 +38,7 @@ Provides-Extra: transformers
|
|
38
38
|
Provides-Extra: vertexai
|
39
39
|
Provides-Extra: watsonx
|
40
40
|
Provides-Extra: weaviate
|
41
|
+
Requires-Dist: aiohttp (>=3.0,<4.0)
|
41
42
|
Requires-Dist: argparse (>=1.0,<2.0)
|
42
43
|
Requires-Dist: backoff (>=2.0,<3.0)
|
43
44
|
Requires-Dist: deprecated (>=1.0,<2.0)
|
@@ -45,35 +46,35 @@ Requires-Dist: jinja2 (>=3.0,<4.0)
|
|
45
46
|
Requires-Dist: opentelemetry-api (>=1.28.0)
|
46
47
|
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.28.0)
|
47
48
|
Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.28.0)
|
48
|
-
Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.33.12) ; extra == "
|
49
|
-
Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.33.12) ; extra == "
|
50
|
-
Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.33.12) ; extra == "
|
51
|
-
Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.33.12) ; extra == "
|
52
|
-
Requires-Dist: opentelemetry-instrumentation-cohere (>=0.33.12) ; extra == "
|
53
|
-
Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.33.12) ; extra == "
|
54
|
-
Requires-Dist: opentelemetry-instrumentation-groq (>=0.33.12) ; extra == "
|
55
|
-
Requires-Dist: opentelemetry-instrumentation-haystack (>=0.33.12) ; extra == "
|
56
|
-
Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.33.12) ; extra == "
|
57
|
-
Requires-Dist: opentelemetry-instrumentation-langchain (>=0.33.12) ; extra == "
|
58
|
-
Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.33.12) ; extra == "
|
59
|
-
Requires-Dist: opentelemetry-instrumentation-marqo (>=0.33.12) ; extra == "
|
60
|
-
Requires-Dist: opentelemetry-instrumentation-milvus (>=0.33.12) ; extra == "
|
61
|
-
Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.33.12) ; extra == "
|
62
|
-
Requires-Dist: opentelemetry-instrumentation-ollama (>=0.33.12) ; extra == "
|
63
|
-
Requires-Dist: opentelemetry-instrumentation-openai (>=0.33.12) ; extra == "
|
64
|
-
Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.33.12) ; extra == "
|
65
|
-
Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.33.12) ; extra == "
|
66
|
-
Requires-Dist: opentelemetry-instrumentation-replicate (>=0.33.12) ; extra == "
|
49
|
+
Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.33.12) ; extra == "all" or extra == "alephalpha"
|
50
|
+
Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.33.12) ; extra == "all" or extra == "anthropic"
|
51
|
+
Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.33.12) ; extra == "all" or extra == "bedrock"
|
52
|
+
Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.33.12) ; extra == "all" or extra == "chromadb"
|
53
|
+
Requires-Dist: opentelemetry-instrumentation-cohere (>=0.33.12) ; extra == "all" or extra == "cohere"
|
54
|
+
Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.33.12) ; extra == "all" or extra == "google-generativeai"
|
55
|
+
Requires-Dist: opentelemetry-instrumentation-groq (>=0.33.12) ; extra == "all" or extra == "groq"
|
56
|
+
Requires-Dist: opentelemetry-instrumentation-haystack (>=0.33.12) ; extra == "all" or extra == "haystack"
|
57
|
+
Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.33.12) ; extra == "all" or extra == "lancedb"
|
58
|
+
Requires-Dist: opentelemetry-instrumentation-langchain (>=0.33.12) ; extra == "all" or extra == "langchain"
|
59
|
+
Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.33.12) ; extra == "all" or extra == "llamaindex"
|
60
|
+
Requires-Dist: opentelemetry-instrumentation-marqo (>=0.33.12) ; extra == "all" or extra == "marqo"
|
61
|
+
Requires-Dist: opentelemetry-instrumentation-milvus (>=0.33.12) ; extra == "all" or extra == "milvus"
|
62
|
+
Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.33.12) ; extra == "all" or extra == "mistralai"
|
63
|
+
Requires-Dist: opentelemetry-instrumentation-ollama (>=0.33.12) ; extra == "all" or extra == "ollama"
|
64
|
+
Requires-Dist: opentelemetry-instrumentation-openai (>=0.33.12) ; extra == "all" or extra == "openai"
|
65
|
+
Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.33.12) ; extra == "all" or extra == "pinecone"
|
66
|
+
Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.33.12) ; extra == "all" or extra == "qdrant"
|
67
|
+
Requires-Dist: opentelemetry-instrumentation-replicate (>=0.33.12) ; extra == "all" or extra == "replicate"
|
67
68
|
Requires-Dist: opentelemetry-instrumentation-requests (>=0.49b0,<0.50)
|
68
|
-
Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.33.12) ; extra == "
|
69
|
+
Requires-Dist: opentelemetry-instrumentation-sagemaker (>=0.33.12) ; extra == "all" or extra == "sagemaker"
|
69
70
|
Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.49b0,<0.50)
|
70
71
|
Requires-Dist: opentelemetry-instrumentation-threading (>=0.49b0,<0.50)
|
71
|
-
Requires-Dist: opentelemetry-instrumentation-together (>=0.33.12) ; extra == "
|
72
|
-
Requires-Dist: opentelemetry-instrumentation-transformers (>=0.33.12) ; extra == "
|
72
|
+
Requires-Dist: opentelemetry-instrumentation-together (>=0.33.12) ; extra == "all" or extra == "together"
|
73
|
+
Requires-Dist: opentelemetry-instrumentation-transformers (>=0.33.12) ; extra == "all" or extra == "transformers"
|
73
74
|
Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.49b0,<0.50)
|
74
|
-
Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.33.12) ; extra == "
|
75
|
-
Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.33.12) ; extra == "
|
76
|
-
Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.33.12) ; extra == "
|
75
|
+
Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.33.12) ; extra == "all" or extra == "vertexai"
|
76
|
+
Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.33.12) ; extra == "all" or extra == "watsonx"
|
77
|
+
Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.33.12) ; extra == "all" or extra == "weaviate"
|
77
78
|
Requires-Dist: opentelemetry-sdk (>=1.28.0)
|
78
79
|
Requires-Dist: opentelemetry-semantic-conventions-ai (==0.4.2)
|
79
80
|
Requires-Dist: pydantic (>=2.7,<3.0)
|
@@ -98,23 +99,28 @@ Check our [open-source repo](https://github.com/lmnr-ai/lmnr) and don't forget t
|
|
98
99
|
|
99
100
|
## Quickstart
|
100
101
|
|
101
|
-
First, install the package
|
102
|
+
First, install the package, specifying the instrumentations you want to use.
|
103
|
+
|
104
|
+
For example, to install the package with OpenAI and Anthropic instrumentations:
|
102
105
|
|
103
106
|
```sh
|
104
|
-
pip install lmnr
|
107
|
+
pip install 'lmnr[anthropic,openai]'
|
108
|
+
```
|
109
|
+
|
110
|
+
To install all possible instrumentations, use the following command:
|
111
|
+
|
112
|
+
```sh
|
113
|
+
pip install 'lmnr[all]'
|
105
114
|
```
|
106
115
|
|
107
|
-
|
116
|
+
Initialize Laminar in your code:
|
108
117
|
|
109
118
|
```python
|
110
|
-
from lmnr import Laminar
|
119
|
+
from lmnr import Laminar
|
111
120
|
|
112
|
-
|
121
|
+
Laminar.initialize(project_api_key="<PROJECT_API_KEY>")
|
113
122
|
```
|
114
123
|
|
115
|
-
This will automatically instrument most of the LLM, Vector DB, and related
|
116
|
-
calls with OpenTelemetry-compatible instrumentation.
|
117
|
-
|
118
124
|
Note that you need to only initialize Laminar once in your application.
|
119
125
|
|
120
126
|
## Instrumentation
|
@@ -127,9 +133,9 @@ This can be useful if you want to trace a request handler or a function which co
|
|
127
133
|
```python
|
128
134
|
import os
|
129
135
|
from openai import OpenAI
|
130
|
-
from lmnr import Laminar
|
136
|
+
from lmnr import Laminar
|
131
137
|
|
132
|
-
|
138
|
+
Laminar.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"])
|
133
139
|
|
134
140
|
client = OpenAI(api_key=os.environ["OPENAI_API_KEY"])
|
135
141
|
|
@@ -152,9 +158,7 @@ def poem_writer(topic: str):
|
|
152
158
|
@observe()
|
153
159
|
def generate_poems():
|
154
160
|
poem1 = poem_writer(topic="laminar flow")
|
155
|
-
L.event("is_poem_generated", True)
|
156
161
|
poem2 = poem_writer(topic="turbulence")
|
157
|
-
L.event("is_poem_generated", True)
|
158
162
|
poems = f"{poem1}\n\n---\n\n{poem2}"
|
159
163
|
return poems
|
160
164
|
```
|
@@ -163,18 +167,10 @@ Also, you can use `Laminar.start_as_current_span` if you want to record a chunk
|
|
163
167
|
|
164
168
|
```python
|
165
169
|
def handle_user_request(topic: str):
|
166
|
-
with
|
167
|
-
...
|
168
|
-
|
170
|
+
with Laminar.start_as_current_span(name="poem_writer", input=topic):
|
169
171
|
poem = poem_writer(topic=topic)
|
170
|
-
|
171
|
-
...
|
172
|
-
|
173
|
-
# while within the span, you can attach laminar events to it
|
174
|
-
L.event("is_poem_generated", True)
|
175
|
-
|
176
172
|
# Use set_span_output to record the output of the span
|
177
|
-
|
173
|
+
Laminar.set_span_output(poem)
|
178
174
|
```
|
179
175
|
|
180
176
|
### Automatic instrumentation
|
@@ -189,9 +185,9 @@ calls with OpenTelemetry-compatible instrumentation, then pass the appropriate i
|
|
189
185
|
For example, if you want to only instrument OpenAI and Anthropic, then do the following:
|
190
186
|
|
191
187
|
```python
|
192
|
-
from lmnr import Laminar
|
188
|
+
from lmnr import Laminar, Instruments
|
193
189
|
|
194
|
-
|
190
|
+
Laminar.initialize(project_api_key=os.environ["LMNR_PROJECT_API_KEY"], instruments={Instruments.OPENAI, Instruments.ANTHROPIC})
|
195
191
|
```
|
196
192
|
|
197
193
|
If you want to fully disable any kind of autoinstrumentation, pass an empty set as `instruments=set()` to `.initialize()`.
|
@@ -272,11 +268,11 @@ Once your pipeline target is set, you can call it from Python in just a few line
|
|
272
268
|
Example use:
|
273
269
|
|
274
270
|
```python
|
275
|
-
from lmnr import Laminar
|
271
|
+
from lmnr import Laminar
|
276
272
|
|
277
|
-
|
273
|
+
Laminar.initialize('<YOUR_PROJECT_API_KEY>', instruments=set())
|
278
274
|
|
279
|
-
result =
|
275
|
+
result = Laminar.run(
|
280
276
|
pipeline = 'my_pipeline_name',
|
281
277
|
inputs = {'input_node_name': 'some_value'},
|
282
278
|
# all environment variables
|
@@ -295,3 +291,15 @@ PipelineRunResponse(
|
|
295
291
|
)
|
296
292
|
```
|
297
293
|
|
294
|
+
## Semantic search
|
295
|
+
|
296
|
+
You can perform a semantic search on a dataset in Laminar by calling `Laminar.semantic_search`.
|
297
|
+
|
298
|
+
```python
|
299
|
+
response = Laminar.semantic_search(
|
300
|
+
query="Greatest Chinese architectural wonders",
|
301
|
+
dataset_id=uuid.UUID("413f8404-724c-4aa4-af16-714d84fd7958"),
|
302
|
+
)
|
303
|
+
```
|
304
|
+
|
305
|
+
[Read more](https://docs.lmnr.ai/datasets/indexing) about indexing and semantic search.
|
@@ -1,5 +1,5 @@
|
|
1
1
|
lmnr/__init__.py,sha256=5cTmg6sY8SYTH24VS-qEdcBwxMpbGkxks8kAqZKgXEU,434
|
2
|
-
lmnr/cli.py,sha256=
|
2
|
+
lmnr/cli.py,sha256=W5zn5DBJiFaVARFy2D-8tJI8ZlGNzjJqk3qiXAG2woo,1456
|
3
3
|
lmnr/openllmetry_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
|
4
4
|
lmnr/openllmetry_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
|
5
5
|
lmnr/openllmetry_sdk/__init__.py,sha256=vVSGTAwUnJvdulHtslkGAd8QCBuv78WUK3bgfBpH6Do,2390
|
@@ -11,22 +11,23 @@ lmnr/openllmetry_sdk/tracing/__init__.py,sha256=xT73L1t2si2CM6QmMiTZ7zn-dKKYBLNr
|
|
11
11
|
lmnr/openllmetry_sdk/tracing/attributes.py,sha256=h970zmb7yTszzf2oHBfOY3cDYhE6O7LhkiHLqa_7x1k,1261
|
12
12
|
lmnr/openllmetry_sdk/tracing/content_allow_list.py,sha256=3feztm6PBWNelc8pAZUcQyEGyeSpNiVKjOaDk65l2ps,846
|
13
13
|
lmnr/openllmetry_sdk/tracing/context_manager.py,sha256=rdSus-p-TaevQ8hIAhfbnZr5dTqRvACDkzXGDpflncY,306
|
14
|
-
lmnr/openllmetry_sdk/tracing/tracing.py,sha256=
|
14
|
+
lmnr/openllmetry_sdk/tracing/tracing.py,sha256=GJM2-VbknimiksqnpI-Zu0KRWFKa93tea0GyONEspkE,31993
|
15
15
|
lmnr/openllmetry_sdk/utils/__init__.py,sha256=pNhf0G3vTd5ccoc03i1MXDbricSaiqCbi1DLWhSekK8,604
|
16
16
|
lmnr/openllmetry_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQvzJk_fZH0OOsRAM1iZQXsR8,2112
|
17
17
|
lmnr/openllmetry_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
|
18
|
-
lmnr/openllmetry_sdk/utils/package_check.py,sha256=
|
18
|
+
lmnr/openllmetry_sdk/utils/package_check.py,sha256=Da4WoTX6J9naODs99DnY9BA-2MxH2pWLmbbVkbQ7VUQ,236
|
19
19
|
lmnr/openllmetry_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
|
20
20
|
lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
21
|
-
lmnr/sdk/datasets.py,sha256=
|
22
|
-
lmnr/sdk/decorators.py,sha256=
|
23
|
-
lmnr/sdk/
|
24
|
-
lmnr/sdk/
|
21
|
+
lmnr/sdk/datasets.py,sha256=KNMp_v3z1ocIltIw7kTgj8o-l9R8N8Tgj0sw1ajQ9C8,1582
|
22
|
+
lmnr/sdk/decorators.py,sha256=ja2EUWUWvFOp28ER0k78PRuxNahwCVyH0TdM3U-xY7U,1856
|
23
|
+
lmnr/sdk/eval_control.py,sha256=G6Fg3Xx_KWv72iBaWlNMdyRTF2bZFQnwJ68sJNSpIcY,177
|
24
|
+
lmnr/sdk/evaluations.py,sha256=gLImD_uB9uXgw07QiJ_OYRTFDGxiPtFCO1c8HyOq2s0,15935
|
25
|
+
lmnr/sdk/laminar.py,sha256=rStB42gKOwz8_0HeNZwrFJ23Qk8qC_nHhu9lpdh73cI,30005
|
25
26
|
lmnr/sdk/log.py,sha256=cZBeUoSK39LMEV-X4-eEhTWOciULRfHaKfRK8YqIM8I,1532
|
26
|
-
lmnr/sdk/types.py,sha256=
|
27
|
+
lmnr/sdk/types.py,sha256=fxRslW0IptnAuWCeFyF_jEZUToyi79-du4kyOcqeSfQ,6352
|
27
28
|
lmnr/sdk/utils.py,sha256=Uk8y15x-sd5tP2ERONahElLDJVEy_3dA_1_5g9A6auY,3358
|
28
|
-
lmnr-0.4.
|
29
|
-
lmnr-0.4.
|
30
|
-
lmnr-0.4.
|
31
|
-
lmnr-0.4.
|
32
|
-
lmnr-0.4.
|
29
|
+
lmnr-0.4.42.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
|
30
|
+
lmnr-0.4.42.dist-info/METADATA,sha256=dZnXnQNNoYgDnPQGUcaqA9aE2ZPr3wwGFczF-jgGk7Y,12350
|
31
|
+
lmnr-0.4.42.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
|
32
|
+
lmnr-0.4.42.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
|
33
|
+
lmnr-0.4.42.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|