langtrace-python-sdk 3.3.23__py3-none-any.whl → 3.3.24__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- examples/awsbedrock_examples/__init__.py +8 -9
- examples/awsbedrock_examples/converse.py +20 -6
- langtrace_python_sdk/instrumentation/aws_bedrock/patch.py +150 -19
- langtrace_python_sdk/version.py +1 -1
- {langtrace_python_sdk-3.3.23.dist-info → langtrace_python_sdk-3.3.24.dist-info}/METADATA +1 -1
- {langtrace_python_sdk-3.3.23.dist-info → langtrace_python_sdk-3.3.24.dist-info}/RECORD +9 -9
- {langtrace_python_sdk-3.3.23.dist-info → langtrace_python_sdk-3.3.24.dist-info}/WHEEL +0 -0
- {langtrace_python_sdk-3.3.23.dist-info → langtrace_python_sdk-3.3.24.dist-info}/entry_points.txt +0 -0
- {langtrace_python_sdk-3.3.23.dist-info → langtrace_python_sdk-3.3.24.dist-info}/licenses/LICENSE +0 -0
@@ -1,9 +1,7 @@
|
|
1
1
|
from examples.awsbedrock_examples.converse import (
|
2
|
-
use_converse_stream,
|
3
|
-
use_converse,
|
4
2
|
use_invoke_model_anthropic,
|
5
|
-
|
6
|
-
|
3
|
+
use_invoke_model_titan,
|
4
|
+
use_invoke_model_llama,
|
7
5
|
)
|
8
6
|
from langtrace_python_sdk import langtrace, with_langtrace_root_span
|
9
7
|
|
@@ -12,8 +10,9 @@ class AWSBedrockRunner:
|
|
12
10
|
@with_langtrace_root_span("AWS_Bedrock")
|
13
11
|
def run(self):
|
14
12
|
|
15
|
-
use_converse_stream()
|
16
|
-
use_converse()
|
17
|
-
use_invoke_model_anthropic()
|
18
|
-
use_invoke_model_cohere()
|
19
|
-
|
13
|
+
# use_converse_stream()
|
14
|
+
# use_converse()
|
15
|
+
# use_invoke_model_anthropic(stream=True)
|
16
|
+
# use_invoke_model_cohere()
|
17
|
+
# use_invoke_model_llama(stream=False)
|
18
|
+
use_invoke_model_titan(stream=False)
|
@@ -88,6 +88,12 @@ def use_invoke_model_titan(stream=False):
|
|
88
88
|
response = brt.invoke_model_with_response_stream(
|
89
89
|
body=body, modelId=modelId, accept=accept, contentType=contentType
|
90
90
|
)
|
91
|
+
# Extract and print the response text in real-time.
|
92
|
+
for event in response["body"]:
|
93
|
+
chunk = json.loads(event["chunk"]["bytes"])
|
94
|
+
if "outputText" in chunk:
|
95
|
+
print(chunk["outputText"], end="")
|
96
|
+
|
91
97
|
else:
|
92
98
|
response = brt.invoke_model(
|
93
99
|
body=body, modelId=modelId, accept=accept, contentType=contentType
|
@@ -130,7 +136,8 @@ def use_invoke_model_anthropic(stream=False):
|
|
130
136
|
for event in stream_response:
|
131
137
|
chunk = event.get("chunk")
|
132
138
|
if chunk:
|
133
|
-
print(json.loads(chunk.get("bytes").decode()))
|
139
|
+
# print(json.loads(chunk.get("bytes").decode()))
|
140
|
+
pass
|
134
141
|
|
135
142
|
else:
|
136
143
|
response = brt.invoke_model(
|
@@ -141,7 +148,7 @@ def use_invoke_model_anthropic(stream=False):
|
|
141
148
|
print(response_body.get("completion"))
|
142
149
|
|
143
150
|
|
144
|
-
def use_invoke_model_llama():
|
151
|
+
def use_invoke_model_llama(stream=False):
|
145
152
|
model_id = "meta.llama3-8b-instruct-v1:0"
|
146
153
|
prompt = "What is the capital of France?"
|
147
154
|
max_gen_len = 128
|
@@ -157,11 +164,18 @@ def use_invoke_model_llama():
|
|
157
164
|
"top_p": top_p,
|
158
165
|
}
|
159
166
|
)
|
160
|
-
response = brt.invoke_model(body=body, modelId=model_id)
|
161
|
-
|
162
|
-
response_body = json.loads(response.get("body").read())
|
163
167
|
|
164
|
-
|
168
|
+
if stream:
|
169
|
+
response = brt.invoke_model_with_response_stream(body=body, modelId=model_id)
|
170
|
+
for event in response["body"]:
|
171
|
+
chunk = json.loads(event["chunk"]["bytes"])
|
172
|
+
if "generation" in chunk:
|
173
|
+
# print(chunk["generation"], end="")
|
174
|
+
pass
|
175
|
+
else:
|
176
|
+
response = brt.invoke_model(body=body, modelId=model_id)
|
177
|
+
response_body = json.loads(response.get("body").read())
|
178
|
+
return response_body
|
165
179
|
|
166
180
|
|
167
181
|
# print(get_foundation_models())
|
@@ -16,9 +16,7 @@ limitations under the License.
|
|
16
16
|
|
17
17
|
import json
|
18
18
|
|
19
|
-
from
|
20
|
-
StreamingWrapper,
|
21
|
-
)
|
19
|
+
from wrapt import ObjectProxy
|
22
20
|
from .stream_body_wrapper import BufferedStreamBody
|
23
21
|
from functools import wraps
|
24
22
|
from langtrace.trace_attributes import (
|
@@ -87,6 +85,11 @@ def patch_aws_bedrock(tracer, version):
|
|
87
85
|
|
88
86
|
client = wrapped(*args, **kwargs)
|
89
87
|
client.invoke_model = patch_invoke_model(client.invoke_model, tracer, version)
|
88
|
+
client.invoke_model_with_response_stream = (
|
89
|
+
patch_invoke_model_with_response_stream(
|
90
|
+
client.invoke_model_with_response_stream, tracer, version
|
91
|
+
)
|
92
|
+
)
|
90
93
|
|
91
94
|
client.converse = patch_converse(client.converse, tracer, version)
|
92
95
|
client.converse_stream = patch_converse_stream(
|
@@ -186,6 +189,56 @@ def patch_invoke_model(original_method, tracer, version):
|
|
186
189
|
return traced_method
|
187
190
|
|
188
191
|
|
192
|
+
def patch_invoke_model_with_response_stream(original_method, tracer, version):
|
193
|
+
@wraps(original_method)
|
194
|
+
def traced_method(*args, **kwargs):
|
195
|
+
modelId = kwargs.get("modelId")
|
196
|
+
(vendor, _) = modelId.split(".")
|
197
|
+
span_attributes = {
|
198
|
+
**get_langtrace_attributes(version, vendor, vendor_type="framework"),
|
199
|
+
**get_extra_attributes(),
|
200
|
+
}
|
201
|
+
span = tracer.start_span(
|
202
|
+
name=get_span_name("aws_bedrock.invoke_model_with_response_stream"),
|
203
|
+
kind=SpanKind.CLIENT,
|
204
|
+
context=set_span_in_context(trace.get_current_span()),
|
205
|
+
)
|
206
|
+
set_span_attributes(span, span_attributes)
|
207
|
+
response = original_method(*args, **kwargs)
|
208
|
+
if span.is_recording():
|
209
|
+
handle_streaming_call(span, kwargs, response)
|
210
|
+
return response
|
211
|
+
|
212
|
+
return traced_method
|
213
|
+
|
214
|
+
|
215
|
+
def handle_streaming_call(span, kwargs, response):
|
216
|
+
|
217
|
+
def stream_finished(response_body):
|
218
|
+
request_body = json.loads(kwargs.get("body"))
|
219
|
+
|
220
|
+
(vendor, model) = kwargs.get("modelId").split(".")
|
221
|
+
|
222
|
+
set_span_attribute(span, SpanAttributes.LLM_REQUEST_MODEL, model)
|
223
|
+
set_span_attribute(span, SpanAttributes.LLM_RESPONSE_MODEL, model)
|
224
|
+
|
225
|
+
if vendor == "amazon":
|
226
|
+
set_amazon_attributes(span, request_body, response_body)
|
227
|
+
|
228
|
+
if vendor == "anthropic":
|
229
|
+
if "prompt" in request_body:
|
230
|
+
set_anthropic_completions_attributes(span, request_body, response_body)
|
231
|
+
elif "messages" in request_body:
|
232
|
+
set_anthropic_messages_attributes(span, request_body, response_body)
|
233
|
+
|
234
|
+
if vendor == "meta":
|
235
|
+
set_llama_meta_attributes(span, request_body, response_body)
|
236
|
+
|
237
|
+
span.end()
|
238
|
+
|
239
|
+
response["body"] = StreamingBedrockWrapper(response["body"], stream_finished)
|
240
|
+
|
241
|
+
|
189
242
|
def handle_call(span, kwargs, response):
|
190
243
|
modelId = kwargs.get("modelId")
|
191
244
|
(vendor, model_name) = modelId.split(".")
|
@@ -195,7 +248,6 @@ def handle_call(span, kwargs, response):
|
|
195
248
|
request_body = json.loads(kwargs.get("body"))
|
196
249
|
response_body = json.loads(response.get("body").read())
|
197
250
|
|
198
|
-
set_span_attribute(span, SpanAttributes.LLM_SYSTEM, vendor)
|
199
251
|
set_span_attribute(span, SpanAttributes.LLM_RESPONSE_MODEL, modelId)
|
200
252
|
set_span_attribute(span, SpanAttributes.LLM_REQUEST_MODEL, modelId)
|
201
253
|
|
@@ -222,12 +274,18 @@ def set_llama_meta_attributes(span, request_body, response_body):
|
|
222
274
|
set_span_attribute(
|
223
275
|
span, SpanAttributes.LLM_REQUEST_MAX_TOKENS, request_body.get("max_gen_len")
|
224
276
|
)
|
277
|
+
if "invocation_metrics" in response_body:
|
278
|
+
input_tokens = response_body.get("invocation_metrics").get("inputTokenCount")
|
279
|
+
output_tokens = response_body.get("invocation_metrics").get("outputTokenCount")
|
280
|
+
else:
|
281
|
+
input_tokens = response_body.get("prompt_token_count")
|
282
|
+
output_tokens = response_body.get("generation_token_count")
|
225
283
|
|
226
284
|
set_usage_attributes(
|
227
285
|
span,
|
228
286
|
{
|
229
|
-
"input_tokens":
|
230
|
-
"output_tokens":
|
287
|
+
"input_tokens": input_tokens,
|
288
|
+
"output_tokens": output_tokens,
|
231
289
|
},
|
232
290
|
)
|
233
291
|
|
@@ -245,7 +303,6 @@ def set_llama_meta_attributes(span, request_body, response_body):
|
|
245
303
|
}
|
246
304
|
]
|
247
305
|
set_span_attribute(span, SpanAttributes.LLM_PROMPTS, json.dumps(prompts))
|
248
|
-
print(completions)
|
249
306
|
set_event_completion(span, completions)
|
250
307
|
|
251
308
|
|
@@ -257,13 +314,22 @@ def set_amazon_attributes(span, request_body, response_body):
|
|
257
314
|
"content": request_body.get("inputText"),
|
258
315
|
}
|
259
316
|
]
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
317
|
+
if "results" in response_body:
|
318
|
+
completions = [
|
319
|
+
{
|
320
|
+
"role": "assistant",
|
321
|
+
"content": result.get("outputText"),
|
322
|
+
}
|
323
|
+
for result in response_body.get("results")
|
324
|
+
]
|
325
|
+
|
326
|
+
else:
|
327
|
+
completions = [
|
328
|
+
{
|
329
|
+
"role": "assistant",
|
330
|
+
"content": response_body.get("outputText"),
|
331
|
+
}
|
332
|
+
]
|
267
333
|
set_span_attribute(
|
268
334
|
span, SpanAttributes.LLM_REQUEST_MAX_TOKENS, config.get("maxTokenCount")
|
269
335
|
)
|
@@ -272,13 +338,19 @@ def set_amazon_attributes(span, request_body, response_body):
|
|
272
338
|
)
|
273
339
|
set_span_attribute(span, SpanAttributes.LLM_REQUEST_TOP_P, config.get("topP"))
|
274
340
|
set_span_attribute(span, SpanAttributes.LLM_PROMPTS, json.dumps(prompts))
|
341
|
+
input_tokens = response_body.get("inputTextTokenCount")
|
342
|
+
if "results" in response_body:
|
343
|
+
output_tokens = sum(
|
344
|
+
int(result.get("tokenCount")) for result in response_body.get("results")
|
345
|
+
)
|
346
|
+
else:
|
347
|
+
output_tokens = response_body.get("outputTextTokenCount")
|
348
|
+
|
275
349
|
set_usage_attributes(
|
276
350
|
span,
|
277
351
|
{
|
278
|
-
"input_tokens":
|
279
|
-
"output_tokens":
|
280
|
-
int(result.get("tokenCount")) for result in response_body.get("results")
|
281
|
-
),
|
352
|
+
"input_tokens": input_tokens,
|
353
|
+
"output_tokens": output_tokens,
|
282
354
|
},
|
283
355
|
)
|
284
356
|
set_event_completion(span, completions)
|
@@ -320,7 +392,7 @@ def set_anthropic_messages_attributes(span, request_body, response_body):
|
|
320
392
|
set_span_attribute(
|
321
393
|
span,
|
322
394
|
SpanAttributes.LLM_REQUEST_MAX_TOKENS,
|
323
|
-
request_body.get("max_tokens_to_sample"),
|
395
|
+
request_body.get("max_tokens_to_sample") or request_body.get("max_tokens"),
|
324
396
|
)
|
325
397
|
set_span_attribute(
|
326
398
|
span,
|
@@ -394,3 +466,62 @@ def set_span_streaming_response(span, response):
|
|
394
466
|
set_event_completion(
|
395
467
|
span, [{"role": role or "assistant", "content": streaming_response}]
|
396
468
|
)
|
469
|
+
|
470
|
+
|
471
|
+
class StreamingBedrockWrapper(ObjectProxy):
|
472
|
+
def __init__(
|
473
|
+
self,
|
474
|
+
response,
|
475
|
+
stream_done_callback=None,
|
476
|
+
):
|
477
|
+
super().__init__(response)
|
478
|
+
|
479
|
+
self._stream_done_callback = stream_done_callback
|
480
|
+
self._accumulating_body = {"generation": ""}
|
481
|
+
|
482
|
+
def __iter__(self):
|
483
|
+
for event in self.__wrapped__:
|
484
|
+
self._process_event(event)
|
485
|
+
yield event
|
486
|
+
|
487
|
+
def _process_event(self, event):
|
488
|
+
chunk = event.get("chunk")
|
489
|
+
if not chunk:
|
490
|
+
return
|
491
|
+
|
492
|
+
decoded_chunk = json.loads(chunk.get("bytes").decode())
|
493
|
+
type = decoded_chunk.get("type")
|
494
|
+
|
495
|
+
if type is None and "outputText" in decoded_chunk:
|
496
|
+
self._stream_done_callback(decoded_chunk)
|
497
|
+
return
|
498
|
+
if "generation" in decoded_chunk:
|
499
|
+
self._accumulating_body["generation"] += decoded_chunk.get("generation")
|
500
|
+
|
501
|
+
if type == "message_start":
|
502
|
+
self._accumulating_body = decoded_chunk.get("message")
|
503
|
+
elif type == "content_block_start":
|
504
|
+
self._accumulating_body["content"].append(
|
505
|
+
decoded_chunk.get("content_block")
|
506
|
+
)
|
507
|
+
elif type == "content_block_delta":
|
508
|
+
self._accumulating_body["content"][-1]["text"] += decoded_chunk.get(
|
509
|
+
"delta"
|
510
|
+
).get("text")
|
511
|
+
|
512
|
+
elif self.has_finished(type, decoded_chunk):
|
513
|
+
self._accumulating_body["invocation_metrics"] = decoded_chunk.get(
|
514
|
+
"amazon-bedrock-invocationMetrics"
|
515
|
+
)
|
516
|
+
self._stream_done_callback(self._accumulating_body)
|
517
|
+
|
518
|
+
def has_finished(self, type, chunk):
|
519
|
+
if type and type == "message_stop":
|
520
|
+
return True
|
521
|
+
|
522
|
+
if "completionReason" in chunk and chunk.get("completionReason") == "FINISH":
|
523
|
+
return True
|
524
|
+
|
525
|
+
if "stop_reason" in chunk and chunk.get("stop_reason") is not None:
|
526
|
+
return True
|
527
|
+
return False
|
langtrace_python_sdk/version.py
CHANGED
@@ -1 +1 @@
|
|
1
|
-
__version__ = "3.3.
|
1
|
+
__version__ = "3.3.24"
|
@@ -3,8 +3,8 @@ examples/anthropic_example/__init__.py,sha256=03us1YuvAJR6fqXX8NH2kROBfTmyz7KzFV
|
|
3
3
|
examples/anthropic_example/completion.py,sha256=3_YEZrt0BLVNJT_RbLXg6JGP2bweuc_HPC2MWR73tOM,713
|
4
4
|
examples/autogen_example/__init__.py,sha256=UJgpzL2yOmzir-DAiGFR1PB1Zz3YcQvYcq5bCN8nl0A,158
|
5
5
|
examples/autogen_example/main.py,sha256=6OJ73VCdHgVrqnekF1S1nK8mXCUABLbUUkQtr7wOCdw,2312
|
6
|
-
examples/awsbedrock_examples/__init__.py,sha256=
|
7
|
-
examples/awsbedrock_examples/converse.py,sha256
|
6
|
+
examples/awsbedrock_examples/__init__.py,sha256=sQ6dq-K2vunY9Y8r8uS-ZwrqKd8XnvV-BukWQ2Gm0fA,537
|
7
|
+
examples/awsbedrock_examples/converse.py,sha256=41CyEla9vFT53ugLJYYHV6kchLEAfXZKMncoJHx1sds,5950
|
8
8
|
examples/azureopenai_example/__init__.py,sha256=PaZM90r6VN4eSOXxb6wGsyhf9-RJCNqBypzk1Xa2GJI,271
|
9
9
|
examples/azureopenai_example/completion.py,sha256=K_GeU0TfJ9lLDfW5VI0Lmm8_I0JXf1x9Qi83ImJ350c,668
|
10
10
|
examples/cerebras_example/__init__.py,sha256=ydfNi0DjFMGVcfo79XVG3VEbzIrHo5wYBgSJzl_asNA,295
|
@@ -111,7 +111,7 @@ examples/weaviate_example/__init__.py,sha256=8JMDBsRSEV10HfTd-YC7xb4txBjD3la56sn
|
|
111
111
|
examples/weaviate_example/query_text.py,sha256=wPHQTc_58kPoKTZMygVjTj-2ZcdrIuaausJfMxNQnQc,127162
|
112
112
|
langtrace_python_sdk/__init__.py,sha256=VZM6i71NR7pBQK6XvJWRelknuTYUhqwqE7PlicKa5Wg,1166
|
113
113
|
langtrace_python_sdk/langtrace.py,sha256=jvfrnkAxc41dmIvwNuVcXwCWyV2vqhGV5qAr5KiMHBA,13239
|
114
|
-
langtrace_python_sdk/version.py,sha256=
|
114
|
+
langtrace_python_sdk/version.py,sha256=BYdEglI7p7QqwHU1MEOvXpBRXVFpeqhDQ3obeVJACVs,23
|
115
115
|
langtrace_python_sdk/constants/__init__.py,sha256=3CNYkWMdd1DrkGqzLUgNZXjdAlM6UFMlf_F-odAToyc,146
|
116
116
|
langtrace_python_sdk/constants/exporter/langtrace_exporter.py,sha256=EVCrouYCpY98f0KSaKr4PzNxPULTZZO6dSA_crEOyJU,106
|
117
117
|
langtrace_python_sdk/constants/instrumentation/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
@@ -147,7 +147,7 @@ langtrace_python_sdk/instrumentation/autogen/patch.py,sha256=7Sq3C8Q5tT27UkWXd1S
|
|
147
147
|
langtrace_python_sdk/instrumentation/aws_bedrock/__init__.py,sha256=IHqPgR1kdDvcoV1nUb-B21PaJ_qbQB0jc011Udi1ioU,96
|
148
148
|
langtrace_python_sdk/instrumentation/aws_bedrock/bedrock_streaming_wrapper.py,sha256=_EMlxfBhjNsB-2TrP0ek-vx1CK5bpgjXPmQWNT9eBcE,1370
|
149
149
|
langtrace_python_sdk/instrumentation/aws_bedrock/instrumentation.py,sha256=M7Dyw1tG18ptD0ctCT9yHzO4UyKCcS8JkHg9WVaz9Ck,1473
|
150
|
-
langtrace_python_sdk/instrumentation/aws_bedrock/patch.py,sha256=
|
150
|
+
langtrace_python_sdk/instrumentation/aws_bedrock/patch.py,sha256=KKNp9gKzoj9YuLrqA0VMDe0FUSBzKtrbXiNN1hdciJM,17941
|
151
151
|
langtrace_python_sdk/instrumentation/aws_bedrock/stream_body_wrapper.py,sha256=ENdhRVHBhdkIlJIc_tkf8ASijUzZdVZM-oonNLdNM48,1584
|
152
152
|
langtrace_python_sdk/instrumentation/cerebras/__init__.py,sha256=9rHNg7PWcZ7a9jExQZlqwWPkvLGcPT-DGWot0_6Bx9k,92
|
153
153
|
langtrace_python_sdk/instrumentation/cerebras/instrumentation.py,sha256=WPsaYxHanYnoxGjDk7fILGJSnSRUs_zoQ30JCyPBMII,1927
|
@@ -277,8 +277,8 @@ tests/pinecone/cassettes/test_query.yaml,sha256=b5v9G3ssUy00oG63PlFUR3JErF2Js-5A
|
|
277
277
|
tests/pinecone/cassettes/test_upsert.yaml,sha256=neWmQ1v3d03V8WoLl8FoFeeCYImb8pxlJBWnFd_lITU,38607
|
278
278
|
tests/qdrant/conftest.py,sha256=9n0uHxxIjWk9fbYc4bx-uP8lSAgLBVx-cV9UjnsyCHM,381
|
279
279
|
tests/qdrant/test_qdrant.py,sha256=pzjAjVY2kmsmGfrI2Gs2xrolfuaNHz7l1fqGQCjp5_o,3353
|
280
|
-
langtrace_python_sdk-3.3.
|
281
|
-
langtrace_python_sdk-3.3.
|
282
|
-
langtrace_python_sdk-3.3.
|
283
|
-
langtrace_python_sdk-3.3.
|
284
|
-
langtrace_python_sdk-3.3.
|
280
|
+
langtrace_python_sdk-3.3.24.dist-info/METADATA,sha256=V0Xhmeqs2wlRHimItoq_pExSCrYd9JPtWQn-F_brXW8,15676
|
281
|
+
langtrace_python_sdk-3.3.24.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
282
|
+
langtrace_python_sdk-3.3.24.dist-info/entry_points.txt,sha256=1_b9-qvf2fE7uQNZcbUei9vLpFZBbbh9LrtGw95ssAo,70
|
283
|
+
langtrace_python_sdk-3.3.24.dist-info/licenses/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
|
284
|
+
langtrace_python_sdk-3.3.24.dist-info/RECORD,,
|
File without changes
|
{langtrace_python_sdk-3.3.23.dist-info → langtrace_python_sdk-3.3.24.dist-info}/entry_points.txt
RENAMED
File without changes
|
{langtrace_python_sdk-3.3.23.dist-info → langtrace_python_sdk-3.3.24.dist-info}/licenses/LICENSE
RENAMED
File without changes
|