lmnr 0.4.6__tar.gz → 0.4.8__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lmnr-0.4.6 → lmnr-0.4.8}/PKG-INFO +40 -3
- lmnr-0.4.8/pyproject.toml +93 -0
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/sdk/decorators.py +2 -7
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/sdk/evaluations.py +4 -4
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/sdk/laminar.py +69 -4
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/sdk/types.py +8 -8
- lmnr-0.4.8/src/lmnr/traceloop_sdk/.flake8 +12 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/.python-version +1 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/README.md +16 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/__init__.py +138 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/config/__init__.py +13 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/decorators/__init__.py +131 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/decorators/base.py +253 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/instruments.py +29 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/metrics/__init__.py +0 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/metrics/metrics.py +176 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/__init__.py +1 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_association_properties/test_langchain_and_external_association_properties.yaml +101 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_association_properties/test_langchain_association_properties.yaml +99 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_manual/test_manual_report.yaml +98 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_manual/test_resource_attributes.yaml +98 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_privacy_no_prompts/test_simple_workflow.yaml +199 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_prompt_management/test_prompt_management.yaml +202 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_sdk_initialization/test_resource_attributes.yaml +199 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_tasks/test_task_io_serialization_with_langchain.yaml +96 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_workflows/test_simple_aworkflow.yaml +98 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_workflows/test_simple_workflow.yaml +199 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/cassettes/test_workflows/test_streaming_workflow.yaml +167 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/conftest.py +111 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/test_association_properties.py +229 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/test_manual.py +48 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/test_nested_tasks.py +47 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/test_privacy_no_prompts.py +50 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/test_sdk_initialization.py +57 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/test_tasks.py +32 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tests/test_workflows.py +261 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tracing/__init__.py +2 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tracing/content_allow_list.py +24 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tracing/context_manager.py +13 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tracing/manual.py +57 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/tracing/tracing.py +1078 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/utils/__init__.py +26 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/utils/in_memory_span_exporter.py +61 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/utils/json_encoder.py +20 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/utils/package_check.py +8 -0
- lmnr-0.4.8/src/lmnr/traceloop_sdk/version.py +1 -0
- lmnr-0.4.6/pyproject.toml +0 -43
- {lmnr-0.4.6 → lmnr-0.4.8}/LICENSE +0 -0
- {lmnr-0.4.6 → lmnr-0.4.8}/README.md +0 -0
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/__init__.py +0 -0
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/sdk/__init__.py +0 -0
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/sdk/log.py +0 -0
- {lmnr-0.4.6 → lmnr-0.4.8}/src/lmnr/sdk/utils.py +0 -0
@@ -1,10 +1,10 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: lmnr
|
3
|
-
Version: 0.4.
|
3
|
+
Version: 0.4.8
|
4
4
|
Summary: Python SDK for Laminar AI
|
5
5
|
License: Apache-2.0
|
6
6
|
Author: lmnr.ai
|
7
|
-
Requires-Python: >=3.9,<4
|
7
|
+
Requires-Python: >=3.9,<4
|
8
8
|
Classifier: License :: OSI Approved :: Apache Software License
|
9
9
|
Classifier: Programming Language :: Python :: 3
|
10
10
|
Classifier: Programming Language :: Python :: 3.9
|
@@ -13,10 +13,47 @@ Classifier: Programming Language :: Python :: 3.11
|
|
13
13
|
Classifier: Programming Language :: Python :: 3.12
|
14
14
|
Requires-Dist: asyncio (>=3.4.3,<4.0.0)
|
15
15
|
Requires-Dist: backoff (>=2.2.1,<3.0.0)
|
16
|
+
Requires-Dist: colorama (>=0.4.6,<0.5.0)
|
17
|
+
Requires-Dist: deprecated (>=1.2.14,<2.0.0)
|
18
|
+
Requires-Dist: jinja2 (>=3.1.2,<4.0.0)
|
19
|
+
Requires-Dist: opentelemetry-api (>=1.27.0,<2.0.0)
|
20
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc (>=1.26.0,<2.0.0)
|
21
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http (>=1.26.0,<2.0.0)
|
22
|
+
Requires-Dist: opentelemetry-instrumentation-alephalpha (>=0.30.0,<0.31.0)
|
23
|
+
Requires-Dist: opentelemetry-instrumentation-anthropic (>=0.30.0,<0.31.0)
|
24
|
+
Requires-Dist: opentelemetry-instrumentation-bedrock (>=0.30.0,<0.31.0)
|
25
|
+
Requires-Dist: opentelemetry-instrumentation-chromadb (>=0.30.0,<0.31.0)
|
26
|
+
Requires-Dist: opentelemetry-instrumentation-cohere (>=0.30.0,<0.31.0)
|
27
|
+
Requires-Dist: opentelemetry-instrumentation-google-generativeai (>=0.30.0,<0.31.0)
|
28
|
+
Requires-Dist: opentelemetry-instrumentation-groq (>=0.30.0,<0.31.0)
|
29
|
+
Requires-Dist: opentelemetry-instrumentation-haystack (>=0.30.0,<0.31.0)
|
30
|
+
Requires-Dist: opentelemetry-instrumentation-lancedb (>=0.30.0,<0.31.0)
|
31
|
+
Requires-Dist: opentelemetry-instrumentation-langchain (>=0.30.0,<0.31.0)
|
32
|
+
Requires-Dist: opentelemetry-instrumentation-llamaindex (>=0.30.0,<0.31.0)
|
33
|
+
Requires-Dist: opentelemetry-instrumentation-marqo (>=0.30.0,<0.31.0)
|
34
|
+
Requires-Dist: opentelemetry-instrumentation-milvus (>=0.30.0,<0.31.0)
|
35
|
+
Requires-Dist: opentelemetry-instrumentation-mistralai (>=0.30.0,<0.31.0)
|
36
|
+
Requires-Dist: opentelemetry-instrumentation-ollama (>=0.30.0,<0.31.0)
|
37
|
+
Requires-Dist: opentelemetry-instrumentation-openai (>=0.30.0,<0.31.0)
|
38
|
+
Requires-Dist: opentelemetry-instrumentation-pinecone (>=0.30.0,<0.31.0)
|
39
|
+
Requires-Dist: opentelemetry-instrumentation-qdrant (>=0.30.0,<0.31.0)
|
40
|
+
Requires-Dist: opentelemetry-instrumentation-replicate (>=0.30.0,<0.31.0)
|
41
|
+
Requires-Dist: opentelemetry-instrumentation-requests (>=0.48b0,<0.49)
|
42
|
+
Requires-Dist: opentelemetry-instrumentation-sqlalchemy (>=0.48b0,<0.49)
|
43
|
+
Requires-Dist: opentelemetry-instrumentation-threading (>=0.48b0,<0.49)
|
44
|
+
Requires-Dist: opentelemetry-instrumentation-together (>=0.30.0,<0.31.0)
|
45
|
+
Requires-Dist: opentelemetry-instrumentation-transformers (>=0.30.0,<0.31.0)
|
46
|
+
Requires-Dist: opentelemetry-instrumentation-urllib3 (>=0.48b0,<0.49)
|
47
|
+
Requires-Dist: opentelemetry-instrumentation-vertexai (>=0.30.0,<0.31.0)
|
48
|
+
Requires-Dist: opentelemetry-instrumentation-watsonx (>=0.30.0,<0.31.0)
|
49
|
+
Requires-Dist: opentelemetry-instrumentation-weaviate (>=0.30.0,<0.31.0)
|
50
|
+
Requires-Dist: opentelemetry-sdk (>=1.27.0,<2.0.0)
|
51
|
+
Requires-Dist: opentelemetry-semantic-conventions-ai (==0.4.1)
|
52
|
+
Requires-Dist: posthog (>3.0.2,<4)
|
16
53
|
Requires-Dist: pydantic (>=2.7.4,<3.0.0)
|
17
54
|
Requires-Dist: python-dotenv (>=1.0.1,<2.0.0)
|
18
55
|
Requires-Dist: requests (>=2.32.3,<3.0.0)
|
19
|
-
Requires-Dist:
|
56
|
+
Requires-Dist: tenacity (>=8.2.3,<9.0.0)
|
20
57
|
Description-Content-Type: text/markdown
|
21
58
|
|
22
59
|
# Laminar Python
|
@@ -0,0 +1,93 @@
|
|
1
|
+
[project]
|
2
|
+
name = "lmnr"
|
3
|
+
version = "0.4.8"
|
4
|
+
description = "Python SDK for Laminar AI"
|
5
|
+
authors = [
|
6
|
+
{ name = "lmnr.ai", email = "founders@lmnr.ai" }
|
7
|
+
]
|
8
|
+
readme = "README.md"
|
9
|
+
requires-python = "^3.9"
|
10
|
+
license = "Apache-2.0"
|
11
|
+
|
12
|
+
[tool.poetry]
|
13
|
+
name = "lmnr"
|
14
|
+
version = "0.4.8"
|
15
|
+
description = "Python SDK for Laminar AI"
|
16
|
+
authors = ["lmnr.ai"]
|
17
|
+
readme = "README.md"
|
18
|
+
license = "Apache-2.0"
|
19
|
+
|
20
|
+
[tool.poetry.dependencies]
|
21
|
+
python = ">=3.9,<4"
|
22
|
+
pydantic = "^2.7.4"
|
23
|
+
requests = "^2.32.3"
|
24
|
+
python-dotenv = "^1.0.1"
|
25
|
+
backoff = "^2.2.1"
|
26
|
+
asyncio = "^3.4.3"
|
27
|
+
opentelemetry-api = "^1.27.0"
|
28
|
+
opentelemetry-sdk = "^1.27.0"
|
29
|
+
opentelemetry-exporter-otlp-proto-http = "^1.26.0"
|
30
|
+
opentelemetry-exporter-otlp-proto-grpc = "^1.26.0"
|
31
|
+
opentelemetry-instrumentation-requests = "^0.48b0"
|
32
|
+
opentelemetry-instrumentation-sqlalchemy = "^0.48b0"
|
33
|
+
opentelemetry-instrumentation-urllib3 = "^0.48b0"
|
34
|
+
opentelemetry-instrumentation-threading = "^0.48b0"
|
35
|
+
opentelemetry-semantic-conventions-ai = "0.4.1"
|
36
|
+
colorama = "^0.4.6"
|
37
|
+
tenacity = "^8.2.3"
|
38
|
+
jinja2 = "^3.1.2"
|
39
|
+
deprecated = "^1.2.14"
|
40
|
+
posthog = ">3.0.2, <4"
|
41
|
+
opentelemetry-instrumentation-mistralai = "^0.30.0"
|
42
|
+
opentelemetry-instrumentation-openai = "^0.30.0"
|
43
|
+
opentelemetry-instrumentation-ollama = "^0.30.0"
|
44
|
+
opentelemetry-instrumentation-anthropic = "^0.30.0"
|
45
|
+
opentelemetry-instrumentation-cohere = "^0.30.0"
|
46
|
+
opentelemetry-instrumentation-google-generativeai = "^0.30.0"
|
47
|
+
opentelemetry-instrumentation-pinecone = "^0.30.0"
|
48
|
+
opentelemetry-instrumentation-qdrant = "^0.30.0"
|
49
|
+
opentelemetry-instrumentation-langchain = "^0.30.0"
|
50
|
+
opentelemetry-instrumentation-lancedb = "^0.30.0"
|
51
|
+
opentelemetry-instrumentation-chromadb = "^0.30.0"
|
52
|
+
opentelemetry-instrumentation-transformers = "^0.30.0"
|
53
|
+
opentelemetry-instrumentation-together = "^0.30.0"
|
54
|
+
opentelemetry-instrumentation-llamaindex = "^0.30.0"
|
55
|
+
opentelemetry-instrumentation-milvus = "^0.30.0"
|
56
|
+
opentelemetry-instrumentation-haystack = "^0.30.0"
|
57
|
+
opentelemetry-instrumentation-bedrock = "^0.30.0"
|
58
|
+
opentelemetry-instrumentation-replicate = "^0.30.0"
|
59
|
+
opentelemetry-instrumentation-vertexai = "^0.30.0"
|
60
|
+
opentelemetry-instrumentation-watsonx = "^0.30.0"
|
61
|
+
opentelemetry-instrumentation-weaviate = "^0.30.0"
|
62
|
+
opentelemetry-instrumentation-alephalpha = "^0.30.0"
|
63
|
+
opentelemetry-instrumentation-marqo = "^0.30.0"
|
64
|
+
opentelemetry-instrumentation-groq = "^0.30.0"
|
65
|
+
|
66
|
+
[tool.poetry.group.dev.dependencies]
|
67
|
+
autopep8 = "^2.2.0"
|
68
|
+
flake8 = "7.0.0"
|
69
|
+
pytest = "^8.2.2"
|
70
|
+
pytest-sugar = "1.0.0"
|
71
|
+
|
72
|
+
[tool.poetry.group.test.dependencies]
|
73
|
+
openai = "^1.31.1"
|
74
|
+
vcrpy = "^6.0.1"
|
75
|
+
pytest-recording = "^0.13.1"
|
76
|
+
pydantic = "<3"
|
77
|
+
pytest-asyncio = "^0.23.7"
|
78
|
+
anthropic = "^0.25.2"
|
79
|
+
langchain = "^0.2.5"
|
80
|
+
langchain-openai = "^0.1.15"
|
81
|
+
|
82
|
+
[build-system]
|
83
|
+
requires = ["poetry-core"]
|
84
|
+
build-backend = "poetry.core.masonry.api"
|
85
|
+
|
86
|
+
[project.entry-points.console_scripts]
|
87
|
+
lmnr = "lmnr.cli.cli:cli"
|
88
|
+
|
89
|
+
[tool.poetry.scripts]
|
90
|
+
lmnr = "lmnr.cli.cli:cli"
|
91
|
+
|
92
|
+
[project.optional-dependencies]
|
93
|
+
test = ["pytest"]
|
@@ -1,9 +1,9 @@
|
|
1
|
-
from
|
1
|
+
from lmnr.traceloop_sdk.decorators.base import (
|
2
2
|
entity_method,
|
3
3
|
aentity_method,
|
4
4
|
)
|
5
5
|
from opentelemetry.trace import INVALID_SPAN, get_current_span
|
6
|
-
from
|
6
|
+
from lmnr.traceloop_sdk import Traceloop
|
7
7
|
|
8
8
|
from typing import Callable, Optional, ParamSpec, TypeVar, cast
|
9
9
|
|
@@ -42,11 +42,6 @@ def observe(
|
|
42
42
|
"""
|
43
43
|
|
44
44
|
def decorator(func: Callable[P, R]) -> Callable[P, R]:
|
45
|
-
if not L.is_initialized():
|
46
|
-
raise Exception(
|
47
|
-
"Laminar is not initialized. Please "
|
48
|
-
+ "call Laminar.initialize() first."
|
49
|
-
)
|
50
45
|
current_span = get_current_span()
|
51
46
|
if current_span != INVALID_SPAN:
|
52
47
|
if session_id is not None:
|
@@ -1,6 +1,6 @@
|
|
1
|
-
from typing import Union
|
1
|
+
from typing import Any, Union
|
2
2
|
|
3
|
-
from .types import
|
3
|
+
from .types import EvaluationDatapoint
|
4
4
|
from .utils import is_async
|
5
5
|
from .laminar import Laminar as L
|
6
6
|
import asyncio
|
@@ -32,8 +32,8 @@ class Evaluation:
|
|
32
32
|
self,
|
33
33
|
name,
|
34
34
|
data: Union[EvaluationDataset, list[Union[EvaluationDatapoint, dict]]],
|
35
|
-
executor:
|
36
|
-
evaluators: list[
|
35
|
+
executor: Any,
|
36
|
+
evaluators: list[Any],
|
37
37
|
batch_size: int = DEFAULT_BATCH_SIZE,
|
38
38
|
project_api_key: str = "",
|
39
39
|
base_url: str = "https://api.lmnr.ai",
|
@@ -4,11 +4,15 @@ from opentelemetry.trace import (
|
|
4
4
|
get_current_span,
|
5
5
|
set_span_in_context,
|
6
6
|
Span,
|
7
|
+
SpanKind,
|
7
8
|
)
|
8
9
|
from opentelemetry.semconv_ai import SpanAttributes
|
9
10
|
from opentelemetry.util.types import AttributeValue
|
10
|
-
from
|
11
|
-
from
|
11
|
+
from opentelemetry.context.context import Context
|
12
|
+
from opentelemetry.util import types
|
13
|
+
from lmnr.traceloop_sdk import Traceloop
|
14
|
+
from lmnr.traceloop_sdk.tracing import get_tracer
|
15
|
+
from contextlib import contextmanager
|
12
16
|
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
|
13
17
|
|
14
18
|
from pydantic.alias_generators import to_snake
|
@@ -96,8 +100,6 @@ class Laminar:
|
|
96
100
|
cls.__initialized = True
|
97
101
|
cls._initialize_logger()
|
98
102
|
Traceloop.init(
|
99
|
-
api_endpoint=cls.__base_url,
|
100
|
-
api_key=cls.__project_api_key,
|
101
103
|
exporter=OTLPSpanExporter(
|
102
104
|
endpoint=cls.__base_url,
|
103
105
|
headers={"authorization": f"Bearer {cls.__project_api_key}"},
|
@@ -289,6 +291,68 @@ class Laminar:
|
|
289
291
|
|
290
292
|
current_span.add_event(name, event)
|
291
293
|
|
294
|
+
@classmethod
|
295
|
+
@contextmanager
|
296
|
+
def start_as_current_span(
|
297
|
+
cls,
|
298
|
+
name: str,
|
299
|
+
input: Any = None,
|
300
|
+
context: Optional[Context] = None,
|
301
|
+
kind: SpanKind = SpanKind.INTERNAL,
|
302
|
+
attributes: types.Attributes = None,
|
303
|
+
links=None,
|
304
|
+
start_time: Optional[int] = None,
|
305
|
+
record_exception: bool = True,
|
306
|
+
set_status_on_exception: bool = True,
|
307
|
+
end_on_exit: bool = True,
|
308
|
+
):
|
309
|
+
"""Start a new span as the current span. Useful for manual instrumentation.
|
310
|
+
This is the preferred and more stable way to use manual instrumentation.
|
311
|
+
|
312
|
+
Usage example:
|
313
|
+
```python
|
314
|
+
with Laminar.start_as_current_span("my_span", input="my_input"):
|
315
|
+
await my_async_function()
|
316
|
+
```
|
317
|
+
|
318
|
+
Args:
|
319
|
+
name (str): name of the span
|
320
|
+
input (Any, optional): input to the span. Will be sent as an
|
321
|
+
attribute, so must be json serializable. Defaults to None.
|
322
|
+
context (Optional[Context], optional): context to start the span in.
|
323
|
+
Defaults to None.
|
324
|
+
kind (SpanKind, optional): kind of the span. Defaults to SpanKind.INTERNAL.
|
325
|
+
attributes (types.Attributes, optional): attributes to set on the span.
|
326
|
+
Defaults to None.
|
327
|
+
links ([type], optional): links to set on the span. Defaults to None.
|
328
|
+
start_time (Optional[int], optional): start time of the span.
|
329
|
+
Defaults to None.
|
330
|
+
record_exception (bool, optional): whether to record exceptions.
|
331
|
+
Defaults to True.
|
332
|
+
set_status_on_exception (bool, optional): whether to set status on exception.
|
333
|
+
Defaults to True.
|
334
|
+
end_on_exit (bool, optional): whether to end the span on exit.
|
335
|
+
Defaults to True.
|
336
|
+
"""
|
337
|
+
with get_tracer() as tracer:
|
338
|
+
with tracer.start_as_current_span(
|
339
|
+
name,
|
340
|
+
context=context,
|
341
|
+
kind=kind,
|
342
|
+
attributes=attributes,
|
343
|
+
links=links,
|
344
|
+
start_time=start_time,
|
345
|
+
record_exception=record_exception,
|
346
|
+
set_status_on_exception=set_status_on_exception,
|
347
|
+
end_on_exit=end_on_exit,
|
348
|
+
) as span:
|
349
|
+
if input is not None:
|
350
|
+
span.set_attribute(
|
351
|
+
SpanAttributes.TRACELOOP_ENTITY_INPUT,
|
352
|
+
json.dumps({"input": input}),
|
353
|
+
)
|
354
|
+
yield span
|
355
|
+
|
292
356
|
@classmethod
|
293
357
|
def start_span(
|
294
358
|
cls,
|
@@ -453,6 +517,7 @@ class Laminar:
|
|
453
517
|
|
454
518
|
@classmethod
|
455
519
|
def _headers(cls):
|
520
|
+
assert cls.__project_api_key is not None, "Project API key is not set"
|
456
521
|
return {
|
457
522
|
"Authorization": "Bearer " + cls.__project_api_key,
|
458
523
|
"Content-Type": "application/json",
|
@@ -90,19 +90,19 @@ class EvaluationDatapoint(pydantic.BaseModel):
|
|
90
90
|
ExecutorFunctionReturnType: TypeAlias = Any
|
91
91
|
EvaluatorFunctionReturnType: TypeAlias = Union[Numeric, dict[str, Numeric]]
|
92
92
|
|
93
|
-
ExecutorFunction: TypeAlias = Callable[
|
94
|
-
|
95
|
-
|
96
|
-
]
|
93
|
+
# ExecutorFunction: TypeAlias = Callable[
|
94
|
+
# [EvaluationDatapointData, *tuple[Any, ...], dict[str, Any]],
|
95
|
+
# Union[ExecutorFunctionReturnType, Awaitable[ExecutorFunctionReturnType]],
|
96
|
+
# ]
|
97
97
|
|
98
98
|
# EvaluatorFunction is a function that takes the output of the executor and the
|
99
99
|
# target data, and returns a score. The score can be a single number or a
|
100
100
|
# record of string keys and number values. The latter is useful for evaluating
|
101
101
|
# multiple criteria in one go instead of running multiple evaluators.
|
102
|
-
EvaluatorFunction: TypeAlias = Callable[
|
103
|
-
|
104
|
-
|
105
|
-
]
|
102
|
+
# EvaluatorFunction: TypeAlias = Callable[
|
103
|
+
# [ExecutorFunctionReturnType, *tuple[Any, ...], dict[str, Any]],
|
104
|
+
# Union[EvaluatorFunctionReturnType, Awaitable[EvaluatorFunctionReturnType]],
|
105
|
+
# ]
|
106
106
|
|
107
107
|
EvaluationStatus: TypeAlias = Literal["Started", "Finished", "Error"]
|
108
108
|
|
@@ -0,0 +1 @@
|
|
1
|
+
3.9.5
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# traceloop-sdk
|
2
|
+
|
3
|
+
Traceloop’s Python SDK allows you to easily start monitoring and debugging your LLM execution. Tracing is done in a non-intrusive way, built on top of OpenTelemetry. You can choose to export the traces to Traceloop, or to your existing observability stack.
|
4
|
+
|
5
|
+
```python
|
6
|
+
Traceloop.init(app_name="joke_generation_service")
|
7
|
+
|
8
|
+
@workflow(name="joke_creation")
|
9
|
+
def create_joke():
|
10
|
+
completion = openai.ChatCompletion.create(
|
11
|
+
model="gpt-3.5-turbo",
|
12
|
+
messages=[{"role": "user", "content": "Tell me a joke about opentelemetry"}],
|
13
|
+
)
|
14
|
+
|
15
|
+
return completion.choices[0].message.content
|
16
|
+
```
|
@@ -0,0 +1,138 @@
|
|
1
|
+
import os
|
2
|
+
import sys
|
3
|
+
from pathlib import Path
|
4
|
+
|
5
|
+
from typing import Optional, Set
|
6
|
+
from colorama import Fore
|
7
|
+
from opentelemetry.sdk.trace import SpanProcessor
|
8
|
+
from opentelemetry.sdk.trace.export import SpanExporter
|
9
|
+
from opentelemetry.sdk.metrics.export import MetricExporter
|
10
|
+
from opentelemetry.sdk.resources import SERVICE_NAME
|
11
|
+
from opentelemetry.propagators.textmap import TextMapPropagator
|
12
|
+
from opentelemetry.util.re import parse_env_headers
|
13
|
+
|
14
|
+
from lmnr.traceloop_sdk.metrics.metrics import MetricsWrapper
|
15
|
+
from lmnr.traceloop_sdk.instruments import Instruments
|
16
|
+
from lmnr.traceloop_sdk.config import (
|
17
|
+
is_content_tracing_enabled,
|
18
|
+
is_tracing_enabled,
|
19
|
+
is_metrics_enabled,
|
20
|
+
)
|
21
|
+
from lmnr.traceloop_sdk.tracing.tracing import (
|
22
|
+
TracerWrapper,
|
23
|
+
set_association_properties,
|
24
|
+
set_external_prompt_tracing_context,
|
25
|
+
)
|
26
|
+
from typing import Dict
|
27
|
+
|
28
|
+
|
29
|
+
class Traceloop:
|
30
|
+
AUTO_CREATED_KEY_PATH = str(
|
31
|
+
Path.home() / ".cache" / "traceloop" / "auto_created_key"
|
32
|
+
)
|
33
|
+
AUTO_CREATED_URL = str(Path.home() / ".cache" / "traceloop" / "auto_created_url")
|
34
|
+
|
35
|
+
__tracer_wrapper: TracerWrapper
|
36
|
+
|
37
|
+
@staticmethod
|
38
|
+
def init(
|
39
|
+
app_name: Optional[str] = sys.argv[0],
|
40
|
+
api_endpoint: str = "https://api.lmnr.ai",
|
41
|
+
api_key: str = None,
|
42
|
+
headers: Dict[str, str] = {},
|
43
|
+
disable_batch=False,
|
44
|
+
exporter: SpanExporter = None,
|
45
|
+
metrics_exporter: MetricExporter = None,
|
46
|
+
metrics_headers: Dict[str, str] = None,
|
47
|
+
processor: SpanProcessor = None,
|
48
|
+
propagator: TextMapPropagator = None,
|
49
|
+
should_enrich_metrics: bool = True,
|
50
|
+
resource_attributes: dict = {},
|
51
|
+
instruments: Optional[Set[Instruments]] = None,
|
52
|
+
) -> None:
|
53
|
+
api_endpoint = os.getenv("TRACELOOP_BASE_URL") or api_endpoint
|
54
|
+
api_key = os.getenv("TRACELOOP_API_KEY") or api_key
|
55
|
+
|
56
|
+
if not is_tracing_enabled():
|
57
|
+
print(Fore.YELLOW + "Tracing is disabled" + Fore.RESET)
|
58
|
+
return
|
59
|
+
|
60
|
+
enable_content_tracing = is_content_tracing_enabled()
|
61
|
+
|
62
|
+
if exporter or processor:
|
63
|
+
print(Fore.GREEN + "Laminar exporting traces to a custom exporter")
|
64
|
+
|
65
|
+
headers = os.getenv("TRACELOOP_HEADERS") or headers
|
66
|
+
|
67
|
+
if isinstance(headers, str):
|
68
|
+
headers = parse_env_headers(headers)
|
69
|
+
|
70
|
+
if (
|
71
|
+
not exporter
|
72
|
+
and not processor
|
73
|
+
and api_endpoint == "https://api.lmnr.ai"
|
74
|
+
and not api_key
|
75
|
+
):
|
76
|
+
print(
|
77
|
+
Fore.RED
|
78
|
+
+ "Error: Missing API key,"
|
79
|
+
+ " go to project settings to create one"
|
80
|
+
)
|
81
|
+
print("Set the LMNR_PROJECT_API_KEY environment variable to the key")
|
82
|
+
print(Fore.RESET)
|
83
|
+
return
|
84
|
+
|
85
|
+
if not exporter and not processor and headers:
|
86
|
+
print(
|
87
|
+
Fore.GREEN
|
88
|
+
+ f"Laminar exporting traces to {api_endpoint}, authenticating with custom headers"
|
89
|
+
)
|
90
|
+
|
91
|
+
if api_key and not exporter and not processor and not headers:
|
92
|
+
print(
|
93
|
+
Fore.GREEN
|
94
|
+
+ f"Laminar exporting traces to {api_endpoint} authenticating with bearer token"
|
95
|
+
)
|
96
|
+
headers = {
|
97
|
+
"Authorization": f"Bearer {api_key}",
|
98
|
+
}
|
99
|
+
|
100
|
+
print(Fore.RESET)
|
101
|
+
|
102
|
+
# Tracer init
|
103
|
+
resource_attributes.update({SERVICE_NAME: app_name})
|
104
|
+
TracerWrapper.set_static_params(
|
105
|
+
resource_attributes, enable_content_tracing, api_endpoint, headers
|
106
|
+
)
|
107
|
+
Traceloop.__tracer_wrapper = TracerWrapper(
|
108
|
+
disable_batch=disable_batch,
|
109
|
+
processor=processor,
|
110
|
+
propagator=propagator,
|
111
|
+
exporter=exporter,
|
112
|
+
should_enrich_metrics=should_enrich_metrics,
|
113
|
+
instruments=instruments,
|
114
|
+
)
|
115
|
+
|
116
|
+
if not metrics_exporter and exporter:
|
117
|
+
return
|
118
|
+
|
119
|
+
metrics_endpoint = os.getenv("TRACELOOP_METRICS_ENDPOINT") or api_endpoint
|
120
|
+
metrics_headers = (
|
121
|
+
os.getenv("TRACELOOP_METRICS_HEADERS") or metrics_headers or headers
|
122
|
+
)
|
123
|
+
|
124
|
+
if not is_metrics_enabled() or not metrics_exporter and exporter:
|
125
|
+
print(Fore.YELLOW + "Metrics are disabled" + Fore.RESET)
|
126
|
+
return
|
127
|
+
|
128
|
+
MetricsWrapper.set_static_params(
|
129
|
+
resource_attributes, metrics_endpoint, metrics_headers
|
130
|
+
)
|
131
|
+
|
132
|
+
Traceloop.__metrics_wrapper = MetricsWrapper(exporter=metrics_exporter)
|
133
|
+
|
134
|
+
def set_association_properties(properties: dict) -> None:
|
135
|
+
set_association_properties(properties)
|
136
|
+
|
137
|
+
def set_prompt(template: str, variables: dict, version: int):
|
138
|
+
set_external_prompt_tracing_context(template, variables, version)
|
@@ -0,0 +1,13 @@
|
|
1
|
+
import os
|
2
|
+
|
3
|
+
|
4
|
+
def is_tracing_enabled() -> bool:
|
5
|
+
return (os.getenv("TRACELOOP_TRACING_ENABLED") or "true").lower() == "true"
|
6
|
+
|
7
|
+
|
8
|
+
def is_content_tracing_enabled() -> bool:
|
9
|
+
return (os.getenv("TRACELOOP_TRACE_CONTENT") or "true").lower() == "true"
|
10
|
+
|
11
|
+
|
12
|
+
def is_metrics_enabled() -> bool:
|
13
|
+
return (os.getenv("TRACELOOP_METRICS_ENABLED") or "true").lower() == "true"
|
@@ -0,0 +1,131 @@
|
|
1
|
+
from typing import Optional
|
2
|
+
|
3
|
+
from opentelemetry.semconv_ai import TraceloopSpanKindValues
|
4
|
+
|
5
|
+
from lmnr.traceloop_sdk.decorators.base import (
|
6
|
+
aentity_class,
|
7
|
+
aentity_method,
|
8
|
+
entity_class,
|
9
|
+
entity_method,
|
10
|
+
)
|
11
|
+
|
12
|
+
|
13
|
+
def task(
|
14
|
+
name: Optional[str] = None,
|
15
|
+
version: Optional[int] = None,
|
16
|
+
method_name: Optional[str] = None,
|
17
|
+
tlp_span_kind: Optional[TraceloopSpanKindValues] = TraceloopSpanKindValues.TASK,
|
18
|
+
):
|
19
|
+
if method_name is None:
|
20
|
+
return entity_method(name=name, version=version, tlp_span_kind=tlp_span_kind)
|
21
|
+
else:
|
22
|
+
return entity_class(
|
23
|
+
name=name,
|
24
|
+
version=version,
|
25
|
+
method_name=method_name,
|
26
|
+
tlp_span_kind=tlp_span_kind,
|
27
|
+
)
|
28
|
+
|
29
|
+
|
30
|
+
def workflow(
|
31
|
+
name: Optional[str] = None,
|
32
|
+
version: Optional[int] = None,
|
33
|
+
method_name: Optional[str] = None,
|
34
|
+
tlp_span_kind: Optional[TraceloopSpanKindValues] = TraceloopSpanKindValues.WORKFLOW,
|
35
|
+
):
|
36
|
+
if method_name is None:
|
37
|
+
return entity_method(name=name, version=version, tlp_span_kind=tlp_span_kind)
|
38
|
+
else:
|
39
|
+
return entity_class(
|
40
|
+
name=name,
|
41
|
+
version=version,
|
42
|
+
method_name=method_name,
|
43
|
+
tlp_span_kind=tlp_span_kind,
|
44
|
+
)
|
45
|
+
|
46
|
+
|
47
|
+
def agent(
|
48
|
+
name: Optional[str] = None,
|
49
|
+
version: Optional[int] = None,
|
50
|
+
method_name: Optional[str] = None,
|
51
|
+
):
|
52
|
+
return workflow(
|
53
|
+
name=name,
|
54
|
+
version=version,
|
55
|
+
method_name=method_name,
|
56
|
+
tlp_span_kind=TraceloopSpanKindValues.AGENT,
|
57
|
+
)
|
58
|
+
|
59
|
+
|
60
|
+
def tool(
|
61
|
+
name: Optional[str] = None,
|
62
|
+
version: Optional[int] = None,
|
63
|
+
method_name: Optional[str] = None,
|
64
|
+
):
|
65
|
+
return task(
|
66
|
+
name=name,
|
67
|
+
version=version,
|
68
|
+
method_name=method_name,
|
69
|
+
tlp_span_kind=TraceloopSpanKindValues.TOOL,
|
70
|
+
)
|
71
|
+
|
72
|
+
|
73
|
+
# Async Decorators
|
74
|
+
def atask(
|
75
|
+
name: Optional[str] = None,
|
76
|
+
version: Optional[int] = None,
|
77
|
+
method_name: Optional[str] = None,
|
78
|
+
tlp_span_kind: Optional[TraceloopSpanKindValues] = TraceloopSpanKindValues.TASK,
|
79
|
+
):
|
80
|
+
if method_name is None:
|
81
|
+
return aentity_method(name=name, version=version, tlp_span_kind=tlp_span_kind)
|
82
|
+
else:
|
83
|
+
return aentity_class(
|
84
|
+
name=name,
|
85
|
+
version=version,
|
86
|
+
method_name=method_name,
|
87
|
+
tlp_span_kind=tlp_span_kind,
|
88
|
+
)
|
89
|
+
|
90
|
+
|
91
|
+
def aworkflow(
|
92
|
+
name: Optional[str] = None,
|
93
|
+
version: Optional[int] = None,
|
94
|
+
method_name: Optional[str] = None,
|
95
|
+
tlp_span_kind: Optional[TraceloopSpanKindValues] = TraceloopSpanKindValues.WORKFLOW,
|
96
|
+
):
|
97
|
+
if method_name is None:
|
98
|
+
return aentity_method(name=name, version=version, tlp_span_kind=tlp_span_kind)
|
99
|
+
else:
|
100
|
+
return aentity_class(
|
101
|
+
name=name,
|
102
|
+
version=version,
|
103
|
+
method_name=method_name,
|
104
|
+
tlp_span_kind=tlp_span_kind,
|
105
|
+
)
|
106
|
+
|
107
|
+
|
108
|
+
def aagent(
|
109
|
+
name: Optional[str] = None,
|
110
|
+
version: Optional[int] = None,
|
111
|
+
method_name: Optional[str] = None,
|
112
|
+
):
|
113
|
+
return atask(
|
114
|
+
name=name,
|
115
|
+
version=version,
|
116
|
+
method_name=method_name,
|
117
|
+
tlp_span_kind=TraceloopSpanKindValues.AGENT,
|
118
|
+
)
|
119
|
+
|
120
|
+
|
121
|
+
def atool(
|
122
|
+
name: Optional[str] = None,
|
123
|
+
version: Optional[int] = None,
|
124
|
+
method_name: Optional[str] = None,
|
125
|
+
):
|
126
|
+
return atask(
|
127
|
+
name=name,
|
128
|
+
version=version,
|
129
|
+
method_name=method_name,
|
130
|
+
tlp_span_kind=TraceloopSpanKindValues.TOOL,
|
131
|
+
)
|