judgeval 0.19.0__py3-none-any.whl → 0.22.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.

This version of judgeval might be problematic. Click here for more details.

Files changed (28) hide show
  1. judgeval/__init__.py +3 -2
  2. judgeval/api/api_types.py +21 -15
  3. judgeval/cli.py +9 -1
  4. judgeval/data/judgment_types.py +21 -20
  5. judgeval/dataset/__init__.py +11 -2
  6. judgeval/env.py +2 -11
  7. judgeval/evaluation/__init__.py +4 -0
  8. judgeval/{prompts/prompt.py → prompt/__init__.py} +30 -20
  9. judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py +1 -1
  10. judgeval/tracer/__init__.py +38 -16
  11. judgeval/tracer/constants.py +1 -1
  12. judgeval/tracer/keys.py +10 -9
  13. judgeval/tracer/llm/llm_anthropic/messages.py +34 -22
  14. judgeval/tracer/llm/llm_anthropic/messages_stream.py +12 -12
  15. judgeval/tracer/llm/llm_google/generate_content.py +8 -6
  16. judgeval/tracer/llm/llm_openai/beta_chat_completions.py +36 -12
  17. judgeval/tracer/llm/llm_openai/chat_completions.py +75 -22
  18. judgeval/tracer/llm/llm_openai/responses.py +77 -22
  19. judgeval/tracer/llm/llm_openai/utils.py +22 -0
  20. judgeval/tracer/llm/llm_together/chat_completions.py +22 -14
  21. judgeval/utils/guards.py +9 -5
  22. judgeval/utils/serialize.py +2 -2
  23. judgeval/version.py +1 -1
  24. {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/METADATA +1 -1
  25. {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/RECORD +28 -27
  26. {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/WHEEL +0 -0
  27. {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/entry_points.txt +0 -0
  28. {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/licenses/LICENSE.md +0 -0
judgeval/tracer/llm/llm_anthropic/messages.py CHANGED
@@ -95,7 +95,7 @@ def _wrap_non_streaming_sync(
95
95
  )
96
96
  ctx["model_name"] = kwargs.get("model", "")
97
97
  set_span_attribute(
98
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
98
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
99
99
  )
100
100
 
101
101
  def post_hook(ctx: Dict[str, Any], result: Message) -> None:
@@ -112,17 +112,19 @@ def _wrap_non_streaming_sync(
112
112
  _extract_anthropic_tokens(result.usage)
113
113
  )
114
114
  set_span_attribute(
115
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
115
+ span,
116
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
117
+ prompt_tokens,
116
118
  )
117
119
  set_span_attribute(
118
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
120
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
119
121
  )
120
122
  set_span_attribute(
121
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
123
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
122
124
  )
123
125
  set_span_attribute(
124
126
  span,
125
- AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
127
+ AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
126
128
  cache_creation,
127
129
  )
128
130
  set_span_attribute(
@@ -133,7 +135,7 @@ def _wrap_non_streaming_sync(
133
135
 
134
136
  set_span_attribute(
135
137
  span,
136
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
138
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
137
139
  result.model,
138
140
  )
139
141
 
@@ -169,7 +171,7 @@ def _wrap_streaming_sync(
169
171
  )
170
172
  ctx["model_name"] = kwargs.get("model", "")
171
173
  set_span_attribute(
172
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
174
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
173
175
  )
174
176
  ctx["accumulated_content"] = ""
175
177
 
@@ -197,17 +199,21 @@ def _wrap_streaming_sync(
197
199
  _extract_anthropic_tokens(usage_data)
198
200
  )
199
201
  set_span_attribute(
200
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
202
+ span,
203
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
204
+ prompt_tokens,
201
205
  )
202
206
  set_span_attribute(
203
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
207
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
204
208
  )
205
209
  set_span_attribute(
206
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
210
+ span,
211
+ AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
212
+ cache_read,
207
213
  )
208
214
  set_span_attribute(
209
215
  span,
210
- AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
216
+ AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
211
217
  cache_creation,
212
218
  )
213
219
  set_span_attribute(
@@ -279,7 +285,7 @@ def _wrap_non_streaming_async(
279
285
  )
280
286
  ctx["model_name"] = kwargs.get("model", "")
281
287
  set_span_attribute(
282
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
288
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
283
289
  )
284
290
 
285
291
  def post_hook(ctx: Dict[str, Any], result: Message) -> None:
@@ -296,17 +302,19 @@ def _wrap_non_streaming_async(
296
302
  _extract_anthropic_tokens(result.usage)
297
303
  )
298
304
  set_span_attribute(
299
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
305
+ span,
306
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
307
+ prompt_tokens,
300
308
  )
301
309
  set_span_attribute(
302
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
310
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
303
311
  )
304
312
  set_span_attribute(
305
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
313
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
306
314
  )
307
315
  set_span_attribute(
308
316
  span,
309
- AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
317
+ AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
310
318
  cache_creation,
311
319
  )
312
320
  set_span_attribute(
@@ -317,7 +325,7 @@ def _wrap_non_streaming_async(
317
325
 
318
326
  set_span_attribute(
319
327
  span,
320
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
328
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
321
329
  result.model,
322
330
  )
323
331
 
@@ -354,7 +362,7 @@ def _wrap_streaming_async(
354
362
  )
355
363
  ctx["model_name"] = kwargs.get("model", "")
356
364
  set_span_attribute(
357
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
365
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
358
366
  )
359
367
  ctx["accumulated_content"] = ""
360
368
 
@@ -382,17 +390,21 @@ def _wrap_streaming_async(
382
390
  _extract_anthropic_tokens(usage_data)
383
391
  )
384
392
  set_span_attribute(
385
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
393
+ span,
394
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
395
+ prompt_tokens,
386
396
  )
387
397
  set_span_attribute(
388
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
398
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
389
399
  )
390
400
  set_span_attribute(
391
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
401
+ span,
402
+ AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
403
+ cache_read,
392
404
  )
393
405
  set_span_attribute(
394
406
  span,
395
- AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
407
+ AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
396
408
  cache_creation,
397
409
  )
398
410
  set_span_attribute(
judgeval/tracer/llm/llm_anthropic/messages_stream.py CHANGED
@@ -44,7 +44,7 @@ def wrap_messages_stream_sync(tracer: Tracer, client: Anthropic) -> None:
44
44
 
45
45
  ctx["model_name"] = kwargs.get("model", "")
46
46
  set_span_attribute(
47
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
47
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
48
48
  )
49
49
  ctx["accumulated_content"] = ""
50
50
 
@@ -125,22 +125,22 @@ def wrap_messages_stream_sync(tracer: Tracer, client: Anthropic) -> None:
125
125
  ) = _extract_anthropic_tokens(final_message.usage)
126
126
  set_span_attribute(
127
127
  span,
128
- AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS,
128
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
129
129
  prompt_tokens,
130
130
  )
131
131
  set_span_attribute(
132
132
  span,
133
- AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS,
133
+ AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS,
134
134
  completion_tokens,
135
135
  )
136
136
  set_span_attribute(
137
137
  span,
138
- AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
138
+ AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
139
139
  cache_read,
140
140
  )
141
141
  set_span_attribute(
142
142
  span,
143
- AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
143
+ AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
144
144
  cache_creation,
145
145
  )
146
146
  set_span_attribute(
@@ -151,7 +151,7 @@ def wrap_messages_stream_sync(tracer: Tracer, client: Anthropic) -> None:
151
151
 
152
152
  set_span_attribute(
153
153
  span,
154
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
154
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
155
155
  final_message.model,
156
156
  )
157
157
  except Exception:
@@ -190,7 +190,7 @@ def wrap_messages_stream_async(tracer: Tracer, client: AsyncAnthropic) -> None:
190
190
 
191
191
  ctx["model_name"] = kwargs.get("model", "")
192
192
  set_span_attribute(
193
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
193
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
194
194
  )
195
195
  ctx["accumulated_content"] = ""
196
196
 
@@ -271,22 +271,22 @@ def wrap_messages_stream_async(tracer: Tracer, client: AsyncAnthropic) -> None:
271
271
  ) = _extract_anthropic_tokens(final_message.usage)
272
272
  set_span_attribute(
273
273
  span,
274
- AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS,
274
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
275
275
  prompt_tokens,
276
276
  )
277
277
  set_span_attribute(
278
278
  span,
279
- AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS,
279
+ AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS,
280
280
  completion_tokens,
281
281
  )
282
282
  set_span_attribute(
283
283
  span,
284
- AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS,
284
+ AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
285
285
  cache_read,
286
286
  )
287
287
  set_span_attribute(
288
288
  span,
289
- AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
289
+ AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
290
290
  cache_creation,
291
291
  )
292
292
  set_span_attribute(
@@ -297,7 +297,7 @@ def wrap_messages_stream_async(tracer: Tracer, client: AsyncAnthropic) -> None:
297
297
 
298
298
  set_span_attribute(
299
299
  span,
300
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
300
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
301
301
  final_message.model,
302
302
  )
303
303
  except Exception:
judgeval/tracer/llm/llm_google/generate_content.py CHANGED
@@ -63,7 +63,7 @@ def wrap_generate_content_sync(tracer: Tracer, client: Client) -> None:
63
63
  )
64
64
  ctx["model_name"] = kwargs.get("model", "")
65
65
  set_span_attribute(
66
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
66
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
67
67
  )
68
68
 
69
69
  def post_hook(ctx: Dict[str, Any], result: GenerateContentResponse) -> None:
@@ -79,17 +79,19 @@ def wrap_generate_content_sync(tracer: Tracer, client: Client) -> None:
79
79
  _extract_google_tokens(usage_data)
80
80
  )
81
81
  set_span_attribute(
82
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
82
+ span,
83
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
84
+ prompt_tokens,
83
85
  )
84
86
  set_span_attribute(
85
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
87
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
86
88
  )
87
89
  set_span_attribute(
88
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
90
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
89
91
  )
90
92
  set_span_attribute(
91
93
  span,
92
- AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS,
94
+ AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
93
95
  cache_creation,
94
96
  )
95
97
  set_span_attribute(
@@ -100,7 +102,7 @@ def wrap_generate_content_sync(tracer: Tracer, client: Client) -> None:
100
102
 
101
103
  set_span_attribute(
102
104
  span,
103
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
105
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
104
106
  result.model_version if result.model_version else ctx["model_name"],
105
107
  )
106
108
 
judgeval/tracer/llm/llm_openai/beta_chat_completions.py CHANGED
@@ -16,6 +16,7 @@ from judgeval.utils.wrappers import (
16
16
  immutable_wrap_sync,
17
17
  immutable_wrap_async,
18
18
  )
19
+ from judgeval.tracer.llm.llm_openai.utils import openai_tokens_converter
19
20
 
20
21
  if TYPE_CHECKING:
21
22
  from judgeval.tracer import Tracer
@@ -45,7 +46,7 @@ def _wrap_beta_non_streaming_sync(
45
46
  )
46
47
  ctx["model_name"] = kwargs.get("model", "")
47
48
  set_span_attribute(
48
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
49
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
49
50
  )
50
51
 
51
52
  def post_hook(ctx: Dict[str, Any], result: ParsedChatCompletion[T]) -> None:
@@ -66,17 +67,29 @@ def _wrap_beta_non_streaming_sync(
66
67
  if prompt_tokens_details:
67
68
  cache_read = prompt_tokens_details.cached_tokens or 0
68
69
 
70
+ prompt_tokens, completion_tokens, cache_read, cache_creation = (
71
+ openai_tokens_converter(
72
+ prompt_tokens,
73
+ completion_tokens,
74
+ cache_read,
75
+ 0,
76
+ usage_data.total_tokens,
77
+ )
78
+ )
79
+
69
80
  set_span_attribute(
70
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
81
+ span,
82
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
83
+ prompt_tokens,
71
84
  )
72
85
  set_span_attribute(
73
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
86
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
74
87
  )
75
88
  set_span_attribute(
76
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
89
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
77
90
  )
78
91
  set_span_attribute(
79
- span, AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
92
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
80
93
  )
81
94
  set_span_attribute(
82
95
  span,
@@ -86,7 +99,7 @@ def _wrap_beta_non_streaming_sync(
86
99
 
87
100
  set_span_attribute(
88
101
  span,
89
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
102
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
90
103
  result.model or ctx["model_name"],
91
104
  )
92
105
 
@@ -128,7 +141,7 @@ def _wrap_beta_non_streaming_async(
128
141
  )
129
142
  ctx["model_name"] = kwargs.get("model", "")
130
143
  set_span_attribute(
131
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
144
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
132
145
  )
133
146
 
134
147
  def post_hook(ctx: Dict[str, Any], result: ParsedChatCompletion[T]) -> None:
@@ -149,17 +162,28 @@ def _wrap_beta_non_streaming_async(
149
162
  if prompt_tokens_details:
150
163
  cache_read = prompt_tokens_details.cached_tokens or 0
151
164
 
165
+ prompt_tokens, completion_tokens, cache_read, cache_creation = (
166
+ openai_tokens_converter(
167
+ prompt_tokens,
168
+ completion_tokens,
169
+ cache_read,
170
+ 0,
171
+ usage_data.total_tokens,
172
+ )
173
+ )
152
174
  set_span_attribute(
153
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
175
+ span,
176
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
177
+ prompt_tokens,
154
178
  )
155
179
  set_span_attribute(
156
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
180
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
157
181
  )
158
182
  set_span_attribute(
159
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
183
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
160
184
  )
161
185
  set_span_attribute(
162
- span, AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
186
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
163
187
  )
164
188
  set_span_attribute(
165
189
  span,
@@ -169,7 +193,7 @@ def _wrap_beta_non_streaming_async(
169
193
 
170
194
  set_span_attribute(
171
195
  span,
172
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
196
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
173
197
  result.model or ctx["model_name"],
174
198
  )
175
199
 
judgeval/tracer/llm/llm_openai/chat_completions.py CHANGED
@@ -25,6 +25,7 @@ from judgeval.utils.wrappers import (
25
25
  immutable_wrap_sync_iterator,
26
26
  immutable_wrap_async_iterator,
27
27
  )
28
+ from judgeval.tracer.llm.llm_openai.utils import openai_tokens_converter
28
29
 
29
30
  if TYPE_CHECKING:
30
31
  from judgeval.tracer import Tracer
@@ -68,7 +69,7 @@ def _wrap_non_streaming_sync(
68
69
  )
69
70
  ctx["model_name"] = kwargs.get("model", "")
70
71
  set_span_attribute(
71
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
72
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
72
73
  )
73
74
 
74
75
  def post_hook(ctx: Dict[str, Any], result: ChatCompletion) -> None:
@@ -89,17 +90,29 @@ def _wrap_non_streaming_sync(
89
90
  if prompt_tokens_details:
90
91
  cache_read = prompt_tokens_details.cached_tokens or 0
91
92
 
93
+ prompt_tokens, completion_tokens, cache_read, cache_creation = (
94
+ openai_tokens_converter(
95
+ prompt_tokens,
96
+ completion_tokens,
97
+ cache_read,
98
+ 0,
99
+ usage_data.total_tokens,
100
+ )
101
+ )
102
+
92
103
  set_span_attribute(
93
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
104
+ span,
105
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
106
+ prompt_tokens,
94
107
  )
95
108
  set_span_attribute(
96
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
109
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
97
110
  )
98
111
  set_span_attribute(
99
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
112
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
100
113
  )
101
114
  set_span_attribute(
102
- span, AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
115
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
103
116
  )
104
117
  set_span_attribute(
105
118
  span,
@@ -109,7 +122,7 @@ def _wrap_non_streaming_sync(
109
122
 
110
123
  set_span_attribute(
111
124
  span,
112
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
125
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
113
126
  result.model or ctx["model_name"],
114
127
  )
115
128
 
@@ -145,7 +158,7 @@ def _wrap_streaming_sync(
145
158
  )
146
159
  ctx["model_name"] = kwargs.get("model", "")
147
160
  set_span_attribute(
148
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
161
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
149
162
  )
150
163
  ctx["accumulated_content"] = ""
151
164
 
@@ -182,17 +195,31 @@ def _wrap_streaming_sync(
182
195
  if chunk.usage.prompt_tokens_details:
183
196
  cache_read = chunk.usage.prompt_tokens_details.cached_tokens or 0
184
197
 
198
+ prompt_tokens, completion_tokens, cache_read, cache_creation = (
199
+ openai_tokens_converter(
200
+ prompt_tokens,
201
+ completion_tokens,
202
+ cache_read,
203
+ 0,
204
+ chunk.usage.total_tokens,
205
+ )
206
+ )
207
+
185
208
  set_span_attribute(
186
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
209
+ span,
210
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
211
+ prompt_tokens,
187
212
  )
188
213
  set_span_attribute(
189
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
214
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
190
215
  )
191
216
  set_span_attribute(
192
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
217
+ span,
218
+ AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
219
+ cache_read,
193
220
  )
194
221
  set_span_attribute(
195
- span, AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
222
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
196
223
  )
197
224
  set_span_attribute(
198
225
  span,
@@ -264,7 +291,7 @@ def _wrap_non_streaming_async(
264
291
  )
265
292
  ctx["model_name"] = kwargs.get("model", "")
266
293
  set_span_attribute(
267
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
294
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
268
295
  )
269
296
 
270
297
  def post_hook(ctx: Dict[str, Any], result: ChatCompletion) -> None:
@@ -285,17 +312,29 @@ def _wrap_non_streaming_async(
285
312
  if prompt_tokens_details:
286
313
  cache_read = prompt_tokens_details.cached_tokens or 0
287
314
 
315
+ prompt_tokens, completion_tokens, cache_read, cache_creation = (
316
+ openai_tokens_converter(
317
+ prompt_tokens,
318
+ completion_tokens,
319
+ cache_read,
320
+ 0,
321
+ usage_data.total_tokens,
322
+ )
323
+ )
324
+
288
325
  set_span_attribute(
289
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
326
+ span,
327
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
328
+ prompt_tokens,
290
329
  )
291
330
  set_span_attribute(
292
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
331
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
293
332
  )
294
333
  set_span_attribute(
295
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
334
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
296
335
  )
297
336
  set_span_attribute(
298
- span, AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
337
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
299
338
  )
300
339
  set_span_attribute(
301
340
  span,
@@ -305,7 +344,7 @@ def _wrap_non_streaming_async(
305
344
 
306
345
  set_span_attribute(
307
346
  span,
308
- AttributeKeys.GEN_AI_RESPONSE_MODEL,
347
+ AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
309
348
  result.model or ctx["model_name"],
310
349
  )
311
350
 
@@ -342,7 +381,7 @@ def _wrap_streaming_async(
342
381
  )
343
382
  ctx["model_name"] = kwargs.get("model", "")
344
383
  set_span_attribute(
345
- ctx["span"], AttributeKeys.GEN_AI_REQUEST_MODEL, ctx["model_name"]
384
+ ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
346
385
  )
347
386
  ctx["accumulated_content"] = ""
348
387
 
@@ -379,17 +418,31 @@ def _wrap_streaming_async(
379
418
  if chunk.usage.prompt_tokens_details:
380
419
  cache_read = chunk.usage.prompt_tokens_details.cached_tokens or 0
381
420
 
421
+ prompt_tokens, completion_tokens, cache_read, cache_creation = (
422
+ openai_tokens_converter(
423
+ prompt_tokens,
424
+ completion_tokens,
425
+ cache_read,
426
+ 0,
427
+ chunk.usage.total_tokens,
428
+ )
429
+ )
430
+
382
431
  set_span_attribute(
383
- span, AttributeKeys.GEN_AI_USAGE_INPUT_TOKENS, prompt_tokens
432
+ span,
433
+ AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
434
+ prompt_tokens,
384
435
  )
385
436
  set_span_attribute(
386
- span, AttributeKeys.GEN_AI_USAGE_OUTPUT_TOKENS, completion_tokens
437
+ span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
387
438
  )
388
439
  set_span_attribute(
389
- span, AttributeKeys.GEN_AI_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
440
+ span,
441
+ AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
442
+ cache_read,
390
443
  )
391
444
  set_span_attribute(
392
- span, AttributeKeys.GEN_AI_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
445
+ span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
393
446
  )
394
447
  set_span_attribute(
395
448
  span,