openlit 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,416 @@
1
+ # pylint: disable=duplicate-code, broad-exception-caught, too-many-statements, unused-argument
2
+ """
3
+ Module for monitoring Mistral API calls.
4
+ """
5
+
6
+ import logging
7
+ from opentelemetry.trace import SpanKind, Status, StatusCode
8
+ from opentelemetry.sdk.resources import TELEMETRY_SDK_NAME
9
+ from openlit.__helpers import get_chat_model_cost, get_embed_model_cost, handle_exception
10
+ from openlit.semcov import SemanticConvetion
11
+
12
+ # Initialize logger for logging potential issues and operations
13
+ logger = logging.getLogger(__name__)
14
+
15
+ def chat(gen_ai_endpoint, version, environment, application_name,
16
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
17
+ """
18
+ Generates a telemetry wrapper for chat to collect metrics.
19
+
20
+ Args:
21
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
22
+ version: Version of the monitoring package.
23
+ environment: Deployment environment (e.g., production, staging).
24
+ application_name: Name of the application using the OpenAI API.
25
+ tracer: OpenTelemetry tracer for creating spans.
26
+ pricing_info: Information used for calculating the cost of OpenAI usage.
27
+ trace_content: Flag indicating whether to trace the actual content.
28
+
29
+ Returns:
30
+ A function that wraps the chat method to add telemetry.
31
+ """
32
+
33
+ def wrapper(wrapped, instance, args, kwargs):
34
+ """
35
+ Wraps the 'chat' API call to add telemetry.
36
+
37
+ This collects metrics such as execution time, cost, and token usage, and handles errors
38
+ gracefully, adding details to the trace for observability.
39
+
40
+ Args:
41
+ wrapped: The original 'chat' method to be wrapped.
42
+ instance: The instance of the class where the original method is defined.
43
+ args: Positional arguments for the 'chat' method.
44
+ kwargs: Keyword arguments for the 'chat' method.
45
+
46
+ Returns:
47
+ The response from the original 'chat' method.
48
+ """
49
+
50
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
51
+ response = wrapped(*args, **kwargs)
52
+
53
+ try:
54
+ # Format 'messages' into a single string
55
+ message_prompt = kwargs.get('messages', "")
56
+ formatted_messages = []
57
+ for message in message_prompt:
58
+ role = message.role
59
+ content = message.content
60
+
61
+ if isinstance(content, list):
62
+ content_str = ", ".join(
63
+ # pylint: disable=line-too-long
64
+ f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
65
+ if 'type' in item else f"text: {item['text']}"
66
+ for item in content
67
+ )
68
+ formatted_messages.append(f"{role}: {content_str}")
69
+ else:
70
+ formatted_messages.append(f"{role}: {content}")
71
+ prompt = " ".join(formatted_messages)
72
+
73
+ # Calculate cost of the operation
74
+ cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
75
+ pricing_info, response.usage.prompt_tokens,
76
+ response.usage.completion_tokens)
77
+
78
+ # Set Span attributes
79
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
80
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
81
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
82
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
83
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
84
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
85
+ gen_ai_endpoint)
86
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
87
+ response.id)
88
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
89
+ environment)
90
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
91
+ application_name)
92
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
93
+ kwargs.get("model", "mistral-small-latest"))
94
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
95
+ kwargs.get("temperature", 0.7))
96
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
97
+ kwargs.get("top_p", 1))
98
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
99
+ kwargs.get("max_tokens", ""))
100
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
101
+ kwargs.get("random_seed", ""))
102
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
103
+ False)
104
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
105
+ response.choices[0].finish_reason)
106
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
107
+ response.usage.prompt_tokens)
108
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
109
+ response.usage.completion_tokens)
110
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
111
+ response.usage.total_tokens)
112
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
113
+ cost)
114
+ if trace_content:
115
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
116
+ prompt)
117
+ # pylint: disable=line-too-long
118
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION, response.choices[0].message.content if response.choices[0].message.content else "")
119
+
120
+ span.set_status(Status(StatusCode.OK))
121
+
122
+ if disable_metrics is False:
123
+ attributes = {
124
+ TELEMETRY_SDK_NAME:
125
+ "openlit",
126
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
127
+ application_name,
128
+ SemanticConvetion.GEN_AI_SYSTEM:
129
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
130
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
131
+ environment,
132
+ SemanticConvetion.GEN_AI_TYPE:
133
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
134
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
135
+ kwargs.get("model", "mistral-small-latest")
136
+ }
137
+
138
+ metrics["genai_requests"].add(1, attributes)
139
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
140
+ metrics["genai_completion_tokens"].add(
141
+ response.usage.completion_tokens, attributes
142
+ )
143
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
144
+ metrics["genai_cost"].record(cost, attributes)
145
+
146
+ # Return original response
147
+ return response
148
+
149
+ except Exception as e:
150
+ handle_exception(span, e)
151
+ logger.error("Error in trace creation: %s", e)
152
+
153
+ # Return original response
154
+ return response
155
+
156
+ return wrapper
157
+
158
+ def chat_stream(gen_ai_endpoint, version, environment, application_name,
159
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
160
+ """
161
+ Generates a telemetry wrapper for chat_stream to collect metrics.
162
+
163
+ Args:
164
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
165
+ version: Version of the monitoring package.
166
+ environment: Deployment environment (e.g., production, staging).
167
+ application_name: Name of the application using the OpenAI API.
168
+ tracer: OpenTelemetry tracer for creating spans.
169
+ pricing_info: Information used for calculating the cost of OpenAI usage.
170
+ trace_content: Flag indicating whether to trace the actual content.
171
+
172
+ Returns:
173
+ A function that wraps the chat method to add telemetry.
174
+ """
175
+
176
+ def wrapper(wrapped, instance, args, kwargs):
177
+ """
178
+ Wraps the 'chat_stream' API call to add telemetry.
179
+
180
+ This collects metrics such as execution time, cost, and token usage, and handles errors
181
+ gracefully, adding details to the trace for observability.
182
+
183
+ Args:
184
+ wrapped: The original 'chat_stream' method to be wrapped.
185
+ instance: The instance of the class where the original method is defined.
186
+ args: Positional arguments for the 'chat_stream' method.
187
+ kwargs: Keyword arguments for the 'chat_stream' method.
188
+
189
+ Returns:
190
+ The response from the original 'chat_stream' method.
191
+ """
192
+
193
+ def stream_generator():
194
+ # pylint: disable=line-too-long
195
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
196
+ # Placeholder for aggregating streaming response
197
+ llmresponse = ""
198
+
199
+ # Loop through streaming events capturing relevant details
200
+ for event in wrapped(*args, **kwargs):
201
+ response_id = event.id
202
+ llmresponse += event.choices[0].delta.content
203
+ if event.usage is not None:
204
+ prompt_tokens = event.usage.prompt_tokens
205
+ completion_tokens = event.usage.completion_tokens
206
+ total_tokens = event.usage.total_tokens
207
+ finish_reason = event.choices[0].finish_reason
208
+ yield event
209
+
210
+ # Handling exception ensure observability without disrupting operation
211
+ try:
212
+ # Format 'messages' into a single string
213
+ message_prompt = kwargs.get('messages', "")
214
+ formatted_messages = []
215
+ for message in message_prompt:
216
+ role = message.role
217
+ content = message.content
218
+
219
+ if isinstance(content, list):
220
+ content_str = ", ".join(
221
+ # pylint: disable=line-too-long
222
+ f"{item['type']}: {item['text'] if 'text' in item else item['image_url']}"
223
+ if 'type' in item else f"text: {item['text']}"
224
+ for item in content
225
+ )
226
+ formatted_messages.append(f"{role}: {content_str}")
227
+ else:
228
+ formatted_messages.append(f"{role}: {content}")
229
+ prompt = " ".join(formatted_messages)
230
+
231
+ # Calculate cost of the operation
232
+ cost = get_chat_model_cost(kwargs.get("model", "mistral-small-latest"),
233
+ pricing_info, prompt_tokens, completion_tokens)
234
+
235
+ # Set Span attributes
236
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
237
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
238
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
239
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
240
+ SemanticConvetion.GEN_AI_TYPE_CHAT)
241
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
242
+ gen_ai_endpoint)
243
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
244
+ response_id)
245
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
246
+ environment)
247
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
248
+ application_name)
249
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
250
+ kwargs.get("model", "mistral-small-latest"))
251
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TEMPERATURE,
252
+ kwargs.get("temperature", 0.7))
253
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_TOP_P,
254
+ kwargs.get("top_p", 1))
255
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MAX_TOKENS,
256
+ kwargs.get("max_tokens", ""))
257
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_SEED,
258
+ kwargs.get("random_seed", ""))
259
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_IS_STREAM,
260
+ True)
261
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_FINISH_REASON,
262
+ finish_reason)
263
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
264
+ prompt_tokens)
265
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COMPLETION_TOKENS,
266
+ completion_tokens)
267
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
268
+ total_tokens)
269
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
270
+ cost)
271
+ if trace_content:
272
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
273
+ prompt)
274
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_COMPLETION,
275
+ llmresponse)
276
+
277
+ span.set_status(Status(StatusCode.OK))
278
+
279
+ if disable_metrics is False:
280
+ attributes = {
281
+ TELEMETRY_SDK_NAME:
282
+ "openlit",
283
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
284
+ application_name,
285
+ SemanticConvetion.GEN_AI_SYSTEM:
286
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
287
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
288
+ environment,
289
+ SemanticConvetion.GEN_AI_TYPE:
290
+ SemanticConvetion.GEN_AI_TYPE_CHAT,
291
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
292
+ kwargs.get("model", "mistral-small-latest")
293
+ }
294
+
295
+ metrics["genai_requests"].add(1, attributes)
296
+ metrics["genai_total_tokens"].add(prompt_tokens + completion_tokens, attributes)
297
+ metrics["genai_completion_tokens"].add(completion_tokens, attributes)
298
+ metrics["genai_prompt_tokens"].add(prompt_tokens, attributes)
299
+ metrics["genai_cost"].record(cost)
300
+
301
+ except Exception as e:
302
+ handle_exception(span, e)
303
+ logger.error("Error in trace creation: %s", e)
304
+
305
+ return stream_generator()
306
+
307
+ return wrapper
308
+
309
+ def embeddings(gen_ai_endpoint, version, environment, application_name,
310
+ tracer, pricing_info, trace_content, metrics, disable_metrics):
311
+ """
312
+ Generates a telemetry wrapper for embeddings to collect metrics.
313
+
314
+ Args:
315
+ gen_ai_endpoint: Endpoint identifier for logging and tracing.
316
+ version: Version of the monitoring package.
317
+ environment: Deployment environment (e.g., production, staging).
318
+ application_name: Name of the application using the OpenAI API.
319
+ tracer: OpenTelemetry tracer for creating spans.
320
+ pricing_info: Information used for calculating the cost of OpenAI usage.
321
+ trace_content: Flag indicating whether to trace the actual content.
322
+
323
+ Returns:
324
+ A function that wraps the embeddings method to add telemetry.
325
+ """
326
+
327
+ def wrapper(wrapped, instance, args, kwargs):
328
+ """
329
+ Wraps the 'embeddings' API call to add telemetry.
330
+
331
+ This collects metrics such as execution time, cost, and token usage, and handles errors
332
+ gracefully, adding details to the trace for observability.
333
+
334
+ Args:
335
+ wrapped: The original 'embeddings' method to be wrapped.
336
+ instance: The instance of the class where the original method is defined.
337
+ args: Positional arguments for the 'embeddings' method.
338
+ kwargs: Keyword arguments for the 'embeddings' method.
339
+
340
+ Returns:
341
+ The response from the original 'embeddings' method.
342
+ """
343
+
344
+ with tracer.start_as_current_span(gen_ai_endpoint, kind= SpanKind.CLIENT) as span:
345
+ response = wrapped(*args, **kwargs)
346
+
347
+ try:
348
+ # Get prompt from kwargs and store as a single string
349
+ prompt = ', '.join(kwargs.get('input', []))
350
+
351
+ # Calculate cost of the operation
352
+ cost = get_embed_model_cost(kwargs.get('model', "mistral-embed"),
353
+ pricing_info, response.usage.prompt_tokens)
354
+
355
+ # Set Span attributes
356
+ span.set_attribute(TELEMETRY_SDK_NAME, "openlit")
357
+ span.set_attribute(SemanticConvetion.GEN_AI_SYSTEM,
358
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL)
359
+ span.set_attribute(SemanticConvetion.GEN_AI_TYPE,
360
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING)
361
+ span.set_attribute(SemanticConvetion.GEN_AI_ENDPOINT,
362
+ gen_ai_endpoint)
363
+ span.set_attribute(SemanticConvetion.GEN_AI_ENVIRONMENT,
364
+ environment)
365
+ span.set_attribute(SemanticConvetion.GEN_AI_APPLICATION_NAME,
366
+ application_name)
367
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_MODEL,
368
+ kwargs.get('model', "mistral-embed"))
369
+ span.set_attribute(SemanticConvetion.GEN_AI_REQUEST_EMBEDDING_FORMAT,
370
+ kwargs.get("encoding_format", "float"))
371
+ span.set_attribute(SemanticConvetion.GEN_AI_RESPONSE_ID,
372
+ response.id)
373
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_PROMPT_TOKENS,
374
+ response.usage.prompt_tokens)
375
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_TOTAL_TOKENS,
376
+ response.usage.total_tokens)
377
+ span.set_attribute(SemanticConvetion.GEN_AI_USAGE_COST,
378
+ cost)
379
+ if trace_content:
380
+ span.set_attribute(SemanticConvetion.GEN_AI_CONTENT_PROMPT,
381
+ prompt)
382
+
383
+ span.set_status(Status(StatusCode.OK))
384
+
385
+ if disable_metrics is False:
386
+ attributes = {
387
+ TELEMETRY_SDK_NAME:
388
+ "openlit",
389
+ SemanticConvetion.GEN_AI_APPLICATION_NAME:
390
+ application_name,
391
+ SemanticConvetion.GEN_AI_SYSTEM:
392
+ SemanticConvetion.GEN_AI_SYSTEM_MISTRAL,
393
+ SemanticConvetion.GEN_AI_ENVIRONMENT:
394
+ environment,
395
+ SemanticConvetion.GEN_AI_TYPE:
396
+ SemanticConvetion.GEN_AI_TYPE_EMBEDDING,
397
+ SemanticConvetion.GEN_AI_REQUEST_MODEL:
398
+ kwargs.get('model', "mistral-embed")
399
+ }
400
+
401
+ metrics["genai_requests"].add(1, attributes)
402
+ metrics["genai_total_tokens"].add(response.usage.total_tokens, attributes)
403
+ metrics["genai_prompt_tokens"].add(response.usage.prompt_tokens, attributes)
404
+ metrics["genai_cost"].record(cost, attributes)
405
+
406
+ # Return original response
407
+ return response
408
+
409
+ except Exception as e:
410
+ handle_exception(span, e)
411
+ logger.error("Error in trace creation: %s", e)
412
+
413
+ # Return original response
414
+ return response
415
+
416
+ return wrapper