lmnr 0.4.11__py3-none-any.whl → 0.4.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lmnr/__init__.py +1 -1
- lmnr/cli.py +39 -0
- lmnr/sdk/decorators.py +3 -2
- lmnr/sdk/evaluations.py +245 -76
- lmnr/sdk/laminar.py +81 -44
- lmnr/sdk/types.py +44 -5
- lmnr/sdk/utils.py +4 -5
- lmnr/traceloop_sdk/__init__.py +3 -42
- lmnr/traceloop_sdk/config/__init__.py +0 -4
- lmnr/traceloop_sdk/decorators/base.py +16 -9
- lmnr/traceloop_sdk/tracing/attributes.py +8 -0
- lmnr/traceloop_sdk/tracing/tracing.py +31 -201
- {lmnr-0.4.11.dist-info → lmnr-0.4.12.dist-info}/METADATA +75 -101
- {lmnr-0.4.11.dist-info → lmnr-0.4.12.dist-info}/RECORD +17 -18
- lmnr-0.4.12.dist-info/entry_points.txt +3 -0
- lmnr/traceloop_sdk/metrics/__init__.py +0 -0
- lmnr/traceloop_sdk/metrics/metrics.py +0 -176
- lmnr/traceloop_sdk/tracing/manual.py +0 -57
- lmnr-0.4.11.dist-info/entry_points.txt +0 -3
- {lmnr-0.4.11.dist-info → lmnr-0.4.12.dist-info}/LICENSE +0 -0
- {lmnr-0.4.11.dist-info → lmnr-0.4.12.dist-info}/WHEEL +0 -0
@@ -1,20 +1,19 @@
|
|
1
|
-
lmnr/__init__.py,sha256=
|
1
|
+
lmnr/__init__.py,sha256=5Ks8UIicCzCBgwSz0MOX3I7jVruPMUO3SmxIwUoODzQ,231
|
2
|
+
lmnr/cli.py,sha256=Ptvm5dsNLKUY5lwnN8XkT5GtCYjzpRNi2WvefknB3OQ,1079
|
2
3
|
lmnr/sdk/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
3
|
-
lmnr/sdk/decorators.py,sha256=
|
4
|
-
lmnr/sdk/evaluations.py,sha256=
|
5
|
-
lmnr/sdk/laminar.py,sha256=
|
4
|
+
lmnr/sdk/decorators.py,sha256=ii7Bqp6flaanIFSK6M1_ZZV-izp4o3hkR1MmY7wnFQQ,2227
|
5
|
+
lmnr/sdk/evaluations.py,sha256=M3LlgYphMEJ-MdZ-UN6jzpWk1G-HHrTmzTQicZoEgwk,13590
|
6
|
+
lmnr/sdk/laminar.py,sha256=3LqzqhsSOHxz11_lxAdvqy_awtOnTdPTeYxYEZ3F4Go,19407
|
6
7
|
lmnr/sdk/log.py,sha256=EgAMY77Zn1bv1imCqrmflD3imoAJ2yveOkIcrIP3e98,1170
|
7
|
-
lmnr/sdk/types.py,sha256=
|
8
|
-
lmnr/sdk/utils.py,sha256=
|
8
|
+
lmnr/sdk/types.py,sha256=QB89q6WeN715x15ukoRVufXk6FSP_1pGn8QsUSIJG5U,5062
|
9
|
+
lmnr/sdk/utils.py,sha256=s81p6uJehgJSaLWy3sR5fTpEDH7vzn3i_UujUHChl6M,3346
|
9
10
|
lmnr/traceloop_sdk/.flake8,sha256=bCxuDlGx3YQ55QHKPiGJkncHanh9qGjQJUujcFa3lAU,150
|
10
11
|
lmnr/traceloop_sdk/.python-version,sha256=9OLQBQVbD4zE4cJsPePhnAfV_snrPSoqEQw-PXgPMOs,6
|
11
|
-
lmnr/traceloop_sdk/__init__.py,sha256=
|
12
|
-
lmnr/traceloop_sdk/config/__init__.py,sha256=
|
12
|
+
lmnr/traceloop_sdk/__init__.py,sha256=hp3q1OsFaGgaQCEanJrL38BJN32hWqCNVCSjYpndEsY,2957
|
13
|
+
lmnr/traceloop_sdk/config/__init__.py,sha256=DliMGp2NjYAqRFLKpWQPUKjGMHRO8QsVfazBA1qENQ8,248
|
13
14
|
lmnr/traceloop_sdk/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
14
|
-
lmnr/traceloop_sdk/decorators/base.py,sha256=
|
15
|
+
lmnr/traceloop_sdk/decorators/base.py,sha256=5YCzAErlhv1bMDO1C9LBlLWYk3bwku0RLjGLR-TkR4c,5128
|
15
16
|
lmnr/traceloop_sdk/instruments.py,sha256=oMvIASueW3GeChpjIdH-DD9aFBVB8OtHZ0HawppTrlI,942
|
16
|
-
lmnr/traceloop_sdk/metrics/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
17
|
-
lmnr/traceloop_sdk/metrics/metrics.py,sha256=AlQ2a2os1WcZbfBd155u_UzBbPrbuPia6O_HbojV9Wc,5055
|
18
17
|
lmnr/traceloop_sdk/tests/__init__.py,sha256=RYnG0-8zbXL0-2Ste1mEBf5sN4d_rQjGTCgPBuaZC74,20
|
19
18
|
lmnr/traceloop_sdk/tests/cassettes/test_association_properties/test_langchain_and_external_association_properties.yaml,sha256=26g0wRA0juicHg_XrhcE8H4vhs1lawDs0o0aLFn-I7w,3103
|
20
19
|
lmnr/traceloop_sdk/tests/cassettes/test_association_properties/test_langchain_association_properties.yaml,sha256=FNlSWlYCsWc3w7UPZzfGjDnxS3gAOhL-kpsu4BTxsDE,3061
|
@@ -36,17 +35,17 @@ lmnr/traceloop_sdk/tests/test_sdk_initialization.py,sha256=fRaf6lrxFzJIN94P1Tav_
|
|
36
35
|
lmnr/traceloop_sdk/tests/test_tasks.py,sha256=xlEx8BKp4yG83SCjK5WkPGfyC33JSrx4h8VyjVwGbgw,906
|
37
36
|
lmnr/traceloop_sdk/tests/test_workflows.py,sha256=RVcfY3WAFIDZC15-aSua21aoQyYeWE7KypDyUsm-2EM,9372
|
38
37
|
lmnr/traceloop_sdk/tracing/__init__.py,sha256=Ckq7zCM26VdJVB5tIZv0GTPyMZKyfso_KWD5yPHaqdo,66
|
38
|
+
lmnr/traceloop_sdk/tracing/attributes.py,sha256=Rvglt_2IeZzKJ-mumrp9qAtTwHza34CrNgv4CvYihk0,221
|
39
39
|
lmnr/traceloop_sdk/tracing/content_allow_list.py,sha256=3feztm6PBWNelc8pAZUcQyEGyeSpNiVKjOaDk65l2ps,846
|
40
40
|
lmnr/traceloop_sdk/tracing/context_manager.py,sha256=csVlB6kDmbgSPsROHwnddvGGblx55v6lJMRj0wsSMQM,304
|
41
|
-
lmnr/traceloop_sdk/tracing/
|
42
|
-
lmnr/traceloop_sdk/tracing/tracing.py,sha256=fDtdZ7mrZIGHDiAAFHIu9x-FRQjHE9b1-KO6yFfGqB0,42463
|
41
|
+
lmnr/traceloop_sdk/tracing/tracing.py,sha256=8plGdX6nErrPERgYXQDQRyBTtVgv2Ies46ph-msLLQE,35443
|
43
42
|
lmnr/traceloop_sdk/utils/__init__.py,sha256=pNhf0G3vTd5ccoc03i1MXDbricSaiqCbi1DLWhSekK8,604
|
44
43
|
lmnr/traceloop_sdk/utils/in_memory_span_exporter.py,sha256=H_4TRaThMO1H6vUQ0OpQvzJk_fZH0OOsRAM1iZQXsR8,2112
|
45
44
|
lmnr/traceloop_sdk/utils/json_encoder.py,sha256=dK6b_axr70IYL7Vv-bu4wntvDDuyntoqsHaddqX7P58,463
|
46
45
|
lmnr/traceloop_sdk/utils/package_check.py,sha256=TZSngzJOpFhfUZLXIs38cpMxQiZSmp0D-sCrIyhz7BA,251
|
47
46
|
lmnr/traceloop_sdk/version.py,sha256=OlatFEFA4ttqSSIiV8jdE-sq3KG5zu2hnC4B4mzWF3s,23
|
48
|
-
lmnr-0.4.
|
49
|
-
lmnr-0.4.
|
50
|
-
lmnr-0.4.
|
51
|
-
lmnr-0.4.
|
52
|
-
lmnr-0.4.
|
47
|
+
lmnr-0.4.12.dist-info/LICENSE,sha256=67b_wJHVV1CBaWkrKFWU1wyqTPSdzH77Ls-59631COg,10411
|
48
|
+
lmnr-0.4.12.dist-info/METADATA,sha256=jlbEUKz0NWRyf5lqRWTGlN88v6beI_lUnoTtcY91q4o,11233
|
49
|
+
lmnr-0.4.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
|
50
|
+
lmnr-0.4.12.dist-info/entry_points.txt,sha256=K1jE20ww4jzHNZLnsfWBvU3YKDGBgbOiYG5Y7ivQcq4,37
|
51
|
+
lmnr-0.4.12.dist-info/RECORD,,
|
File without changes
|
@@ -1,176 +0,0 @@
|
|
1
|
-
from collections.abc import Sequence
|
2
|
-
from typing import Dict
|
3
|
-
|
4
|
-
from opentelemetry.exporter.otlp.proto.grpc.metric_exporter import (
|
5
|
-
OTLPMetricExporter as GRPCExporter,
|
6
|
-
)
|
7
|
-
from opentelemetry.exporter.otlp.proto.http.metric_exporter import (
|
8
|
-
OTLPMetricExporter as HTTPExporter,
|
9
|
-
)
|
10
|
-
from opentelemetry.semconv_ai import Meters
|
11
|
-
from opentelemetry.sdk.metrics import MeterProvider
|
12
|
-
from opentelemetry.sdk.metrics.export import (
|
13
|
-
PeriodicExportingMetricReader,
|
14
|
-
MetricExporter,
|
15
|
-
)
|
16
|
-
from opentelemetry.sdk.metrics.view import View, ExplicitBucketHistogramAggregation
|
17
|
-
from opentelemetry.sdk.resources import Resource
|
18
|
-
|
19
|
-
from opentelemetry import metrics
|
20
|
-
|
21
|
-
|
22
|
-
class MetricsWrapper(object):
|
23
|
-
resource_attributes: dict = {}
|
24
|
-
endpoint: str = None
|
25
|
-
# if it needs headers?
|
26
|
-
headers: Dict[str, str] = {}
|
27
|
-
__metrics_exporter: MetricExporter = None
|
28
|
-
__metrics_provider: MeterProvider = None
|
29
|
-
|
30
|
-
def __new__(cls, exporter: MetricExporter = None) -> "MetricsWrapper":
|
31
|
-
if not hasattr(cls, "instance"):
|
32
|
-
obj = cls.instance = super(MetricsWrapper, cls).__new__(cls)
|
33
|
-
if not MetricsWrapper.endpoint:
|
34
|
-
return obj
|
35
|
-
|
36
|
-
obj.__metrics_exporter = (
|
37
|
-
exporter
|
38
|
-
if exporter
|
39
|
-
else init_metrics_exporter(
|
40
|
-
MetricsWrapper.endpoint, MetricsWrapper.headers
|
41
|
-
)
|
42
|
-
)
|
43
|
-
|
44
|
-
obj.__metrics_provider = init_metrics_provider(
|
45
|
-
obj.__metrics_exporter, MetricsWrapper.resource_attributes
|
46
|
-
)
|
47
|
-
|
48
|
-
return cls.instance
|
49
|
-
|
50
|
-
@staticmethod
|
51
|
-
def set_static_params(
|
52
|
-
resource_attributes: dict,
|
53
|
-
endpoint: str,
|
54
|
-
headers: Dict[str, str],
|
55
|
-
) -> None:
|
56
|
-
MetricsWrapper.resource_attributes = resource_attributes
|
57
|
-
MetricsWrapper.endpoint = endpoint
|
58
|
-
MetricsWrapper.headers = headers
|
59
|
-
|
60
|
-
|
61
|
-
def init_metrics_exporter(endpoint: str, headers: Dict[str, str]) -> MetricExporter:
|
62
|
-
if "http" in endpoint.lower() or "https" in endpoint.lower():
|
63
|
-
return HTTPExporter(endpoint=f"{endpoint}/v1/metrics", headers=headers)
|
64
|
-
else:
|
65
|
-
return GRPCExporter(endpoint=endpoint, headers=headers)
|
66
|
-
|
67
|
-
|
68
|
-
def init_metrics_provider(
|
69
|
-
exporter: MetricExporter, resource_attributes: dict = None
|
70
|
-
) -> MeterProvider:
|
71
|
-
resource = (
|
72
|
-
Resource.create(resource_attributes)
|
73
|
-
if resource_attributes
|
74
|
-
else Resource.create()
|
75
|
-
)
|
76
|
-
reader = PeriodicExportingMetricReader(exporter)
|
77
|
-
provider = MeterProvider(
|
78
|
-
metric_readers=[reader],
|
79
|
-
resource=resource,
|
80
|
-
views=metric_views(),
|
81
|
-
)
|
82
|
-
|
83
|
-
metrics.set_meter_provider(provider)
|
84
|
-
return provider
|
85
|
-
|
86
|
-
|
87
|
-
def metric_views() -> Sequence[View]:
|
88
|
-
return [
|
89
|
-
View(
|
90
|
-
instrument_name=Meters.LLM_TOKEN_USAGE,
|
91
|
-
aggregation=ExplicitBucketHistogramAggregation(
|
92
|
-
[
|
93
|
-
0.01,
|
94
|
-
0.02,
|
95
|
-
0.04,
|
96
|
-
0.08,
|
97
|
-
0.16,
|
98
|
-
0.32,
|
99
|
-
0.64,
|
100
|
-
1.28,
|
101
|
-
2.56,
|
102
|
-
5.12,
|
103
|
-
10.24,
|
104
|
-
20.48,
|
105
|
-
40.96,
|
106
|
-
81.92,
|
107
|
-
]
|
108
|
-
),
|
109
|
-
),
|
110
|
-
View(
|
111
|
-
instrument_name=Meters.LLM_OPERATION_DURATION,
|
112
|
-
aggregation=ExplicitBucketHistogramAggregation(
|
113
|
-
[
|
114
|
-
1,
|
115
|
-
4,
|
116
|
-
16,
|
117
|
-
64,
|
118
|
-
256,
|
119
|
-
1024,
|
120
|
-
4096,
|
121
|
-
16384,
|
122
|
-
65536,
|
123
|
-
262144,
|
124
|
-
1048576,
|
125
|
-
4194304,
|
126
|
-
16777216,
|
127
|
-
67108864,
|
128
|
-
]
|
129
|
-
),
|
130
|
-
),
|
131
|
-
View(
|
132
|
-
instrument_name=Meters.PINECONE_DB_QUERY_DURATION,
|
133
|
-
aggregation=ExplicitBucketHistogramAggregation(
|
134
|
-
[
|
135
|
-
0.01,
|
136
|
-
0.02,
|
137
|
-
0.04,
|
138
|
-
0.08,
|
139
|
-
0.16,
|
140
|
-
0.32,
|
141
|
-
0.64,
|
142
|
-
1.28,
|
143
|
-
2.56,
|
144
|
-
5.12,
|
145
|
-
10.24,
|
146
|
-
20.48,
|
147
|
-
40.96,
|
148
|
-
81.92,
|
149
|
-
]
|
150
|
-
),
|
151
|
-
),
|
152
|
-
View(
|
153
|
-
instrument_name=Meters.PINECONE_DB_QUERY_SCORES,
|
154
|
-
aggregation=ExplicitBucketHistogramAggregation(
|
155
|
-
[
|
156
|
-
-1,
|
157
|
-
-0.875,
|
158
|
-
-0.75,
|
159
|
-
-0.625,
|
160
|
-
-0.5,
|
161
|
-
-0.375,
|
162
|
-
-0.25,
|
163
|
-
-0.125,
|
164
|
-
0,
|
165
|
-
0.125,
|
166
|
-
0.25,
|
167
|
-
0.375,
|
168
|
-
0.5,
|
169
|
-
0.625,
|
170
|
-
0.75,
|
171
|
-
0.875,
|
172
|
-
1,
|
173
|
-
]
|
174
|
-
),
|
175
|
-
),
|
176
|
-
]
|
@@ -1,57 +0,0 @@
|
|
1
|
-
from contextlib import contextmanager
|
2
|
-
from opentelemetry.semconv_ai import SpanAttributes
|
3
|
-
from opentelemetry.trace import Span
|
4
|
-
from pydantic import BaseModel
|
5
|
-
from lmnr.traceloop_sdk.tracing.context_manager import get_tracer
|
6
|
-
|
7
|
-
|
8
|
-
class LLMMessage(BaseModel):
|
9
|
-
role: str
|
10
|
-
content: str
|
11
|
-
|
12
|
-
|
13
|
-
class LLMUsage(BaseModel):
|
14
|
-
prompt_tokens: int
|
15
|
-
completion_tokens: int
|
16
|
-
total_tokens: int
|
17
|
-
|
18
|
-
|
19
|
-
class LLMSpan:
|
20
|
-
_span: Span = None
|
21
|
-
|
22
|
-
def __init__(self, span: Span):
|
23
|
-
self._span = span
|
24
|
-
pass
|
25
|
-
|
26
|
-
def report_request(self, model: str, messages: list[LLMMessage]):
|
27
|
-
self._span.set_attribute(SpanAttributes.LLM_REQUEST_MODEL, model)
|
28
|
-
for idx, message in enumerate(messages):
|
29
|
-
self._span.set_attribute(
|
30
|
-
f"{SpanAttributes.LLM_PROMPTS}.{idx}.role", message.role
|
31
|
-
)
|
32
|
-
self._span.set_attribute(
|
33
|
-
f"{SpanAttributes.LLM_PROMPTS}.{idx}.content", message.content
|
34
|
-
)
|
35
|
-
|
36
|
-
def report_response(self, model: str, completions: list[str]):
|
37
|
-
self._span.set_attribute(SpanAttributes.LLM_RESPONSE_MODEL, model)
|
38
|
-
for idx, completion in enumerate(completions):
|
39
|
-
self._span.set_attribute(
|
40
|
-
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}.role", "assistant"
|
41
|
-
)
|
42
|
-
self._span.set_attribute(
|
43
|
-
f"{SpanAttributes.LLM_COMPLETIONS}.{idx}", completion
|
44
|
-
)
|
45
|
-
|
46
|
-
|
47
|
-
@contextmanager
|
48
|
-
def track_llm_call(vendor: str, type: str):
|
49
|
-
with get_tracer() as tracer:
|
50
|
-
with tracer.start_as_current_span(name=f"{vendor}.{type}") as span:
|
51
|
-
span.set_attribute(SpanAttributes.LLM_SYSTEM, vendor)
|
52
|
-
span.set_attribute(SpanAttributes.LLM_REQUEST_TYPE, type)
|
53
|
-
llm_span = LLMSpan(span)
|
54
|
-
try:
|
55
|
-
yield llm_span
|
56
|
-
finally:
|
57
|
-
span.end()
|
File without changes
|
File without changes
|