judgeval 0.19.0__py3-none-any.whl → 0.22.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of judgeval might be problematic. Click here for more details.
- judgeval/__init__.py +3 -2
- judgeval/api/api_types.py +21 -15
- judgeval/cli.py +9 -1
- judgeval/data/judgment_types.py +21 -20
- judgeval/dataset/__init__.py +11 -2
- judgeval/env.py +2 -11
- judgeval/evaluation/__init__.py +4 -0
- judgeval/{prompts/prompt.py → prompt/__init__.py} +30 -20
- judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py +1 -1
- judgeval/tracer/__init__.py +38 -16
- judgeval/tracer/constants.py +1 -1
- judgeval/tracer/keys.py +10 -9
- judgeval/tracer/llm/llm_anthropic/messages.py +34 -22
- judgeval/tracer/llm/llm_anthropic/messages_stream.py +12 -12
- judgeval/tracer/llm/llm_google/generate_content.py +8 -6
- judgeval/tracer/llm/llm_openai/beta_chat_completions.py +36 -12
- judgeval/tracer/llm/llm_openai/chat_completions.py +75 -22
- judgeval/tracer/llm/llm_openai/responses.py +77 -22
- judgeval/tracer/llm/llm_openai/utils.py +22 -0
- judgeval/tracer/llm/llm_together/chat_completions.py +22 -14
- judgeval/utils/guards.py +9 -5
- judgeval/utils/serialize.py +2 -2
- judgeval/version.py +1 -1
- {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/METADATA +1 -1
- {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/RECORD +28 -27
- {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/WHEEL +0 -0
- {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/entry_points.txt +0 -0
- {judgeval-0.19.0.dist-info → judgeval-0.22.0.dist-info}/licenses/LICENSE.md +0 -0
|
@@ -24,6 +24,7 @@ from judgeval.utils.wrappers import (
|
|
|
24
24
|
immutable_wrap_sync_iterator,
|
|
25
25
|
immutable_wrap_async_iterator,
|
|
26
26
|
)
|
|
27
|
+
from judgeval.tracer.llm.llm_openai.utils import openai_tokens_converter
|
|
27
28
|
|
|
28
29
|
if TYPE_CHECKING:
|
|
29
30
|
from judgeval.tracer import Tracer
|
|
@@ -62,7 +63,7 @@ def _wrap_responses_non_streaming_sync(
|
|
|
62
63
|
)
|
|
63
64
|
ctx["model_name"] = kwargs.get("model", "")
|
|
64
65
|
set_span_attribute(
|
|
65
|
-
ctx["span"], AttributeKeys.
|
|
66
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
|
|
66
67
|
)
|
|
67
68
|
|
|
68
69
|
def post_hook(ctx: Dict[str, Any], result: Response) -> None:
|
|
@@ -80,17 +81,29 @@ def _wrap_responses_non_streaming_sync(
|
|
|
80
81
|
completion_tokens = usage_data.output_tokens or 0
|
|
81
82
|
cache_read = usage_data.input_tokens_details.cached_tokens or 0
|
|
82
83
|
|
|
84
|
+
prompt_tokens, completion_tokens, cache_read, cache_creation = (
|
|
85
|
+
openai_tokens_converter(
|
|
86
|
+
prompt_tokens,
|
|
87
|
+
completion_tokens,
|
|
88
|
+
cache_read,
|
|
89
|
+
0,
|
|
90
|
+
usage_data.total_tokens,
|
|
91
|
+
)
|
|
92
|
+
)
|
|
93
|
+
|
|
83
94
|
set_span_attribute(
|
|
84
|
-
span,
|
|
95
|
+
span,
|
|
96
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
97
|
+
prompt_tokens,
|
|
85
98
|
)
|
|
86
99
|
set_span_attribute(
|
|
87
|
-
span, AttributeKeys.
|
|
100
|
+
span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
|
|
88
101
|
)
|
|
89
102
|
set_span_attribute(
|
|
90
|
-
span, AttributeKeys.
|
|
103
|
+
span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
|
|
91
104
|
)
|
|
92
105
|
set_span_attribute(
|
|
93
|
-
span, AttributeKeys.
|
|
106
|
+
span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
|
|
94
107
|
)
|
|
95
108
|
set_span_attribute(
|
|
96
109
|
span,
|
|
@@ -101,7 +114,7 @@ def _wrap_responses_non_streaming_sync(
|
|
|
101
114
|
if hasattr(result, "model"):
|
|
102
115
|
set_span_attribute(
|
|
103
116
|
span,
|
|
104
|
-
AttributeKeys.
|
|
117
|
+
AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
|
|
105
118
|
result.model or ctx["model_name"],
|
|
106
119
|
)
|
|
107
120
|
|
|
@@ -137,7 +150,7 @@ def _wrap_responses_streaming_sync(
|
|
|
137
150
|
)
|
|
138
151
|
ctx["model_name"] = kwargs.get("model", "")
|
|
139
152
|
set_span_attribute(
|
|
140
|
-
ctx["span"], AttributeKeys.
|
|
153
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
|
|
141
154
|
)
|
|
142
155
|
ctx["accumulated_content"] = ""
|
|
143
156
|
|
|
@@ -167,6 +180,7 @@ def _wrap_responses_streaming_sync(
|
|
|
167
180
|
):
|
|
168
181
|
prompt_tokens = chunk.response.usage.input_tokens or 0
|
|
169
182
|
completion_tokens = chunk.response.usage.output_tokens or 0
|
|
183
|
+
total_tokens = chunk.response.usage.total_tokens or 0
|
|
170
184
|
# Safely access nested cached_tokens
|
|
171
185
|
input_tokens_details = getattr(
|
|
172
186
|
chunk.response.usage, "input_tokens_details", None
|
|
@@ -177,21 +191,35 @@ def _wrap_responses_streaming_sync(
|
|
|
177
191
|
else 0
|
|
178
192
|
)
|
|
179
193
|
|
|
194
|
+
prompt_tokens, completion_tokens, cache_read, cache_creation = (
|
|
195
|
+
openai_tokens_converter(
|
|
196
|
+
prompt_tokens,
|
|
197
|
+
completion_tokens,
|
|
198
|
+
cache_read,
|
|
199
|
+
0,
|
|
200
|
+
total_tokens,
|
|
201
|
+
)
|
|
202
|
+
)
|
|
203
|
+
|
|
180
204
|
set_span_attribute(
|
|
181
|
-
span,
|
|
205
|
+
span,
|
|
206
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
207
|
+
prompt_tokens,
|
|
182
208
|
)
|
|
183
209
|
set_span_attribute(
|
|
184
210
|
span,
|
|
185
|
-
AttributeKeys.
|
|
211
|
+
AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS,
|
|
186
212
|
completion_tokens,
|
|
187
213
|
)
|
|
188
214
|
set_span_attribute(
|
|
189
215
|
span,
|
|
190
|
-
AttributeKeys.
|
|
216
|
+
AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
|
|
191
217
|
cache_read,
|
|
192
218
|
)
|
|
193
219
|
set_span_attribute(
|
|
194
|
-
span,
|
|
220
|
+
span,
|
|
221
|
+
AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
|
|
222
|
+
0,
|
|
195
223
|
)
|
|
196
224
|
set_span_attribute(
|
|
197
225
|
span,
|
|
@@ -266,7 +294,7 @@ def _wrap_responses_non_streaming_async(
|
|
|
266
294
|
)
|
|
267
295
|
ctx["model_name"] = kwargs.get("model", "")
|
|
268
296
|
set_span_attribute(
|
|
269
|
-
ctx["span"], AttributeKeys.
|
|
297
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
|
|
270
298
|
)
|
|
271
299
|
|
|
272
300
|
def post_hook(ctx: Dict[str, Any], result: Response) -> None:
|
|
@@ -284,17 +312,29 @@ def _wrap_responses_non_streaming_async(
|
|
|
284
312
|
completion_tokens = usage_data.output_tokens or 0
|
|
285
313
|
cache_read = usage_data.input_tokens_details.cached_tokens or 0
|
|
286
314
|
|
|
315
|
+
prompt_tokens, completion_tokens, cache_read, cache_creation = (
|
|
316
|
+
openai_tokens_converter(
|
|
317
|
+
prompt_tokens,
|
|
318
|
+
completion_tokens,
|
|
319
|
+
cache_read,
|
|
320
|
+
0,
|
|
321
|
+
usage_data.total_tokens,
|
|
322
|
+
)
|
|
323
|
+
)
|
|
324
|
+
|
|
287
325
|
set_span_attribute(
|
|
288
|
-
span,
|
|
326
|
+
span,
|
|
327
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
328
|
+
prompt_tokens,
|
|
289
329
|
)
|
|
290
330
|
set_span_attribute(
|
|
291
|
-
span, AttributeKeys.
|
|
331
|
+
span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
|
|
292
332
|
)
|
|
293
333
|
set_span_attribute(
|
|
294
|
-
span, AttributeKeys.
|
|
334
|
+
span, AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS, cache_read
|
|
295
335
|
)
|
|
296
336
|
set_span_attribute(
|
|
297
|
-
span, AttributeKeys.
|
|
337
|
+
span, AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS, 0
|
|
298
338
|
)
|
|
299
339
|
set_span_attribute(
|
|
300
340
|
span,
|
|
@@ -305,7 +345,7 @@ def _wrap_responses_non_streaming_async(
|
|
|
305
345
|
if hasattr(result, "model"):
|
|
306
346
|
set_span_attribute(
|
|
307
347
|
span,
|
|
308
|
-
AttributeKeys.
|
|
348
|
+
AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
|
|
309
349
|
result.model or ctx["model_name"],
|
|
310
350
|
)
|
|
311
351
|
|
|
@@ -341,7 +381,7 @@ def _wrap_responses_streaming_async(
|
|
|
341
381
|
)
|
|
342
382
|
ctx["model_name"] = kwargs.get("model", "")
|
|
343
383
|
set_span_attribute(
|
|
344
|
-
ctx["span"], AttributeKeys.
|
|
384
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, ctx["model_name"]
|
|
345
385
|
)
|
|
346
386
|
ctx["accumulated_content"] = ""
|
|
347
387
|
|
|
@@ -373,6 +413,7 @@ def _wrap_responses_streaming_async(
|
|
|
373
413
|
):
|
|
374
414
|
prompt_tokens = chunk.response.usage.input_tokens or 0
|
|
375
415
|
completion_tokens = chunk.response.usage.output_tokens or 0
|
|
416
|
+
total_tokens = chunk.response.usage.total_tokens or 0
|
|
376
417
|
# Safely access nested cached_tokens
|
|
377
418
|
input_tokens_details = getattr(
|
|
378
419
|
chunk.response.usage, "input_tokens_details", None
|
|
@@ -383,21 +424,35 @@ def _wrap_responses_streaming_async(
|
|
|
383
424
|
else 0
|
|
384
425
|
)
|
|
385
426
|
|
|
427
|
+
prompt_tokens, completion_tokens, cache_read, cache_creation = (
|
|
428
|
+
openai_tokens_converter(
|
|
429
|
+
prompt_tokens,
|
|
430
|
+
completion_tokens,
|
|
431
|
+
cache_read,
|
|
432
|
+
0,
|
|
433
|
+
total_tokens,
|
|
434
|
+
)
|
|
435
|
+
)
|
|
436
|
+
|
|
386
437
|
set_span_attribute(
|
|
387
|
-
span,
|
|
438
|
+
span,
|
|
439
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
440
|
+
prompt_tokens,
|
|
388
441
|
)
|
|
389
442
|
set_span_attribute(
|
|
390
443
|
span,
|
|
391
|
-
AttributeKeys.
|
|
444
|
+
AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS,
|
|
392
445
|
completion_tokens,
|
|
393
446
|
)
|
|
394
447
|
set_span_attribute(
|
|
395
448
|
span,
|
|
396
|
-
AttributeKeys.
|
|
449
|
+
AttributeKeys.JUDGMENT_USAGE_CACHE_READ_INPUT_TOKENS,
|
|
397
450
|
cache_read,
|
|
398
451
|
)
|
|
399
452
|
set_span_attribute(
|
|
400
|
-
span,
|
|
453
|
+
span,
|
|
454
|
+
AttributeKeys.JUDGMENT_USAGE_CACHE_CREATION_INPUT_TOKENS,
|
|
455
|
+
0,
|
|
401
456
|
)
|
|
402
457
|
set_span_attribute(
|
|
403
458
|
span,
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
def openai_tokens_converter(
|
|
2
|
+
prompt_tokens: int,
|
|
3
|
+
completion_tokens: int,
|
|
4
|
+
cache_read: int,
|
|
5
|
+
cache_creation: int,
|
|
6
|
+
total_tokens: int,
|
|
7
|
+
) -> tuple[int, int, int, int]:
|
|
8
|
+
"""
|
|
9
|
+
Returns:
|
|
10
|
+
tuple[int, int, int, int]:
|
|
11
|
+
- judgment.usage.non_cached_input
|
|
12
|
+
- judgment.usage.output_tokens
|
|
13
|
+
- judgment.usage.cached_input_tokens
|
|
14
|
+
- judgment.usage.cache_creation_tokens
|
|
15
|
+
"""
|
|
16
|
+
manual_tokens = prompt_tokens + completion_tokens + cache_read + cache_creation
|
|
17
|
+
|
|
18
|
+
if manual_tokens > total_tokens:
|
|
19
|
+
# This is the openAI case where we need to subtract the cached tokens from the input tokens
|
|
20
|
+
return prompt_tokens - cache_read, completion_tokens, cache_read, cache_creation
|
|
21
|
+
else:
|
|
22
|
+
return prompt_tokens, completion_tokens, cache_read, cache_creation
|
|
@@ -73,7 +73,7 @@ def _wrap_non_streaming_sync(
|
|
|
73
73
|
)
|
|
74
74
|
ctx["model_name"] = prefixed_model_name
|
|
75
75
|
set_span_attribute(
|
|
76
|
-
ctx["span"], AttributeKeys.
|
|
76
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, prefixed_model_name
|
|
77
77
|
)
|
|
78
78
|
|
|
79
79
|
def post_hook(ctx: Dict[str, Any], result: ChatCompletionResponse) -> None:
|
|
@@ -90,10 +90,12 @@ def _wrap_non_streaming_sync(
|
|
|
90
90
|
result.usage
|
|
91
91
|
)
|
|
92
92
|
set_span_attribute(
|
|
93
|
-
span,
|
|
93
|
+
span,
|
|
94
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
95
|
+
prompt_tokens,
|
|
94
96
|
)
|
|
95
97
|
set_span_attribute(
|
|
96
|
-
span, AttributeKeys.
|
|
98
|
+
span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
|
|
97
99
|
)
|
|
98
100
|
set_span_attribute(
|
|
99
101
|
span,
|
|
@@ -103,7 +105,7 @@ def _wrap_non_streaming_sync(
|
|
|
103
105
|
|
|
104
106
|
set_span_attribute(
|
|
105
107
|
span,
|
|
106
|
-
AttributeKeys.
|
|
108
|
+
AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
|
|
107
109
|
ctx["model_name"],
|
|
108
110
|
)
|
|
109
111
|
|
|
@@ -143,7 +145,7 @@ def _wrap_streaming_sync(
|
|
|
143
145
|
)
|
|
144
146
|
ctx["model_name"] = prefixed_model_name
|
|
145
147
|
set_span_attribute(
|
|
146
|
-
ctx["span"], AttributeKeys.
|
|
148
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, prefixed_model_name
|
|
147
149
|
)
|
|
148
150
|
ctx["accumulated_content"] = ""
|
|
149
151
|
|
|
@@ -171,10 +173,12 @@ def _wrap_streaming_sync(
|
|
|
171
173
|
chunk.usage
|
|
172
174
|
)
|
|
173
175
|
set_span_attribute(
|
|
174
|
-
span,
|
|
176
|
+
span,
|
|
177
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
178
|
+
prompt_tokens,
|
|
175
179
|
)
|
|
176
180
|
set_span_attribute(
|
|
177
|
-
span, AttributeKeys.
|
|
181
|
+
span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
|
|
178
182
|
)
|
|
179
183
|
set_span_attribute(
|
|
180
184
|
span,
|
|
@@ -249,7 +253,7 @@ def _wrap_non_streaming_async(
|
|
|
249
253
|
)
|
|
250
254
|
ctx["model_name"] = prefixed_model_name
|
|
251
255
|
set_span_attribute(
|
|
252
|
-
ctx["span"], AttributeKeys.
|
|
256
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, prefixed_model_name
|
|
253
257
|
)
|
|
254
258
|
|
|
255
259
|
def post_hook(ctx: Dict[str, Any], result: ChatCompletionResponse) -> None:
|
|
@@ -266,10 +270,12 @@ def _wrap_non_streaming_async(
|
|
|
266
270
|
result.usage
|
|
267
271
|
)
|
|
268
272
|
set_span_attribute(
|
|
269
|
-
span,
|
|
273
|
+
span,
|
|
274
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
275
|
+
prompt_tokens,
|
|
270
276
|
)
|
|
271
277
|
set_span_attribute(
|
|
272
|
-
span, AttributeKeys.
|
|
278
|
+
span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
|
|
273
279
|
)
|
|
274
280
|
set_span_attribute(
|
|
275
281
|
span,
|
|
@@ -279,7 +285,7 @@ def _wrap_non_streaming_async(
|
|
|
279
285
|
|
|
280
286
|
set_span_attribute(
|
|
281
287
|
span,
|
|
282
|
-
AttributeKeys.
|
|
288
|
+
AttributeKeys.JUDGMENT_LLM_MODEL_NAME,
|
|
283
289
|
ctx["model_name"],
|
|
284
290
|
)
|
|
285
291
|
|
|
@@ -320,7 +326,7 @@ def _wrap_streaming_async(
|
|
|
320
326
|
)
|
|
321
327
|
ctx["model_name"] = prefixed_model_name
|
|
322
328
|
set_span_attribute(
|
|
323
|
-
ctx["span"], AttributeKeys.
|
|
329
|
+
ctx["span"], AttributeKeys.JUDGMENT_LLM_MODEL_NAME, prefixed_model_name
|
|
324
330
|
)
|
|
325
331
|
ctx["accumulated_content"] = ""
|
|
326
332
|
|
|
@@ -348,10 +354,12 @@ def _wrap_streaming_async(
|
|
|
348
354
|
chunk.usage
|
|
349
355
|
)
|
|
350
356
|
set_span_attribute(
|
|
351
|
-
span,
|
|
357
|
+
span,
|
|
358
|
+
AttributeKeys.JUDGMENT_USAGE_NON_CACHED_INPUT_TOKENS,
|
|
359
|
+
prompt_tokens,
|
|
352
360
|
)
|
|
353
361
|
set_span_attribute(
|
|
354
|
-
span, AttributeKeys.
|
|
362
|
+
span, AttributeKeys.JUDGMENT_USAGE_OUTPUT_TOKENS, completion_tokens
|
|
355
363
|
)
|
|
356
364
|
set_span_attribute(
|
|
357
365
|
span,
|
judgeval/utils/guards.py
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
from __future__ import annotations
|
|
2
2
|
|
|
3
3
|
from typing import TYPE_CHECKING
|
|
4
|
+
from judgeval.logger import judgeval_logger
|
|
4
5
|
|
|
5
6
|
if TYPE_CHECKING:
|
|
6
7
|
from typing import TypeVar
|
|
@@ -8,24 +9,27 @@ if TYPE_CHECKING:
|
|
|
8
9
|
T = TypeVar("T")
|
|
9
10
|
|
|
10
11
|
|
|
11
|
-
def expect_exists(value: T | None, message: str) -> T:
|
|
12
|
-
if value
|
|
13
|
-
|
|
12
|
+
def expect_exists(value: T | None, message: str, default: T) -> T:
|
|
13
|
+
if not value:
|
|
14
|
+
judgeval_logger.error(message)
|
|
15
|
+
return default
|
|
14
16
|
|
|
15
17
|
return value
|
|
16
18
|
|
|
17
19
|
|
|
18
|
-
def expect_api_key(api_key: str | None) -> str:
|
|
20
|
+
def expect_api_key(api_key: str | None) -> str | None:
|
|
19
21
|
return expect_exists(
|
|
20
22
|
api_key,
|
|
21
23
|
"API Key is not set, please set JUDGMENT_API_KEY in the environment variables or pass it as `api_key`",
|
|
24
|
+
default=None,
|
|
22
25
|
)
|
|
23
26
|
|
|
24
27
|
|
|
25
|
-
def expect_organization_id(organization_id: str | None) -> str:
|
|
28
|
+
def expect_organization_id(organization_id: str | None) -> str | None:
|
|
26
29
|
return expect_exists(
|
|
27
30
|
organization_id,
|
|
28
31
|
"Organization ID is not set, please set JUDGMENT_ORG_ID in the environment variables or pass it as `organization_id`",
|
|
32
|
+
default=None,
|
|
29
33
|
)
|
|
30
34
|
|
|
31
35
|
|
judgeval/utils/serialize.py
CHANGED
|
@@ -247,7 +247,7 @@ encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE)
|
|
|
247
247
|
# Seralize arbitrary object to a json string
|
|
248
248
|
def safe_serialize(obj: Any) -> str:
|
|
249
249
|
try:
|
|
250
|
-
return orjson.dumps(json_encoder(obj)).decode()
|
|
250
|
+
return orjson.dumps(json_encoder(obj), option=orjson.OPT_NON_STR_KEYS).decode()
|
|
251
251
|
except Exception as e:
|
|
252
252
|
judgeval_logger.warning(f"Error serializing object: {e}")
|
|
253
|
-
return
|
|
253
|
+
return repr(obj)
|
judgeval/version.py
CHANGED
|
@@ -1,24 +1,24 @@
|
|
|
1
|
-
judgeval/__init__.py,sha256=
|
|
2
|
-
judgeval/cli.py,sha256=
|
|
1
|
+
judgeval/__init__.py,sha256=RRiBbXUj7M1VW3NqFvMZlXyI72duh3VA5bfIWqPmKNw,6670
|
|
2
|
+
judgeval/cli.py,sha256=T9nKO9eHMOiLCgxaxuihqtRHsG_dMT06sW6X873MmnI,2209
|
|
3
3
|
judgeval/constants.py,sha256=JZZJ1MqzZZDVk-5PRPRbmLnM8mXI-RDL5vxa1JFuscs,3408
|
|
4
|
-
judgeval/env.py,sha256=
|
|
4
|
+
judgeval/env.py,sha256=uFggNNKmfDaa5dmZMwwXVIDdHAHe524jDWUpByV4hm4,1879
|
|
5
5
|
judgeval/exceptions.py,sha256=tTbfe4yoOtPXmn22UQz9-6a-5PT9uOko85xaRRwr0Sw,621
|
|
6
6
|
judgeval/logger.py,sha256=VP5blbsJ53mvJbNHfBf5p2KrARUrkrErpPkB-__Hh3U,1562
|
|
7
|
-
judgeval/version.py,sha256=
|
|
7
|
+
judgeval/version.py,sha256=JT1Ltu1j1xi9sHf7Inhosvl-1tp0QPms04o2RCvwJUI,74
|
|
8
8
|
judgeval/warnings.py,sha256=LbGte14ppiFjrkp-JJYueZ40NWFvMkWRvPXr6r-fUWw,73
|
|
9
9
|
judgeval/api/__init__.py,sha256=dGZm9KtgLMnmbiyDEJ_D7suuVqmsibR_Cd0YZRJ7qHI,15210
|
|
10
|
-
judgeval/api/api_types.py,sha256=
|
|
10
|
+
judgeval/api/api_types.py,sha256=PJ5ZQWuvCl5GXFzhcpOw6Iuktr50lo5BaILmZcAKWfc,10085
|
|
11
11
|
judgeval/data/__init__.py,sha256=1tU0EN0ThIfQ1fad5I3dKxAfTcZ5U8cvTLcQ6qLVLU0,407
|
|
12
12
|
judgeval/data/evaluation_run.py,sha256=O41p99wNAuCAf6lsLNKzkZ6W-kL9LlzCYxVls7IcKkA,4727
|
|
13
13
|
judgeval/data/example.py,sha256=eGJpF-lyUH734Cg90B7WtU9f8iKoS3VFGeV6R-GVCCc,1039
|
|
14
|
-
judgeval/data/judgment_types.py,sha256=
|
|
14
|
+
judgeval/data/judgment_types.py,sha256=7RsrB2FvnsRRtnqMMfQzAMMn9oNvA076hbE2tmzKNXc,18874
|
|
15
15
|
judgeval/data/result.py,sha256=XufFGSAkBDfevPUmzSgsR9HEqytISkM0U5HkhJmsjpY,2102
|
|
16
16
|
judgeval/data/scorer_data.py,sha256=HeP15ZgftFTJCF8JmDJCLWXRnZJIaGDJCzl7Hg6gWwE,2006
|
|
17
17
|
judgeval/data/trace.py,sha256=zSiR3o6xt8Z46XA3M9fJBtViF0BsPO6yKp9jxdscOSc,3881
|
|
18
18
|
judgeval/data/scripts/fix_default_factory.py,sha256=lvp2JwYZqz-XpD9LZNa3mANZVP-jJSZoNzolI6JWERM,591
|
|
19
19
|
judgeval/data/scripts/openapi_transform.py,sha256=Sm04JClzyP1ga8KA3gkIdsae8Hlx-XU7-x0gHCQYOhg,3877
|
|
20
|
-
judgeval/dataset/__init__.py,sha256=
|
|
21
|
-
judgeval/evaluation/__init__.py,sha256=
|
|
20
|
+
judgeval/dataset/__init__.py,sha256=s7HuBH_TQOLZ1arqaY2QRiSp-4mI_fF_9OykK_1QbsI,8858
|
|
21
|
+
judgeval/evaluation/__init__.py,sha256=e9H4h73MINpcBlBYpkXiUaoCdWxnzvaYK0Ob0awY-kM,13064
|
|
22
22
|
judgeval/integrations/langgraph/__init__.py,sha256=HwXmtDxaO75Kn4KPErnMb6Ne6FcpRxV_SCYVuwFsve0,332
|
|
23
23
|
judgeval/integrations/openlit/__init__.py,sha256=-8D4D6-fGsWPwoOojw82OaE9X5sUbmb16x1bF-WfOmg,1571
|
|
24
24
|
judgeval/judges/__init__.py,sha256=e7JnTc1TG_SwqydDHTXHIP0EBazQxt-ydMQG7ghSU5A,228
|
|
@@ -26,7 +26,7 @@ judgeval/judges/base_judge.py,sha256=_dz0qWsKRxzXxpRY9l6mrxTRYPSF2FE4ZXkrzhZ4gbY
|
|
|
26
26
|
judgeval/judges/litellm_judge.py,sha256=5vEF0IUo7HVWnOF2ww-DMke8Xkarnz32B_qbgKjc0-I,4182
|
|
27
27
|
judgeval/judges/together_judge.py,sha256=GzwlXZJzle8hT-vWKmq39JyIeanJqJfHDOkrksUbzk0,4398
|
|
28
28
|
judgeval/judges/utils.py,sha256=ITbYwvjU3o9-FIAReFvxh24yJrx9LV3l9BnSBgKUpxg,2068
|
|
29
|
-
judgeval/
|
|
29
|
+
judgeval/prompt/__init__.py,sha256=Qgrd8u4WniaOjbRAoEFEeMnTmaqIGx5ZGX_U85iqhs0,11010
|
|
30
30
|
judgeval/scorers/__init__.py,sha256=pomKzEy4YNFyygYp8vbS3co8iB5CMstRkQwdUgi1u4g,744
|
|
31
31
|
judgeval/scorers/agent_scorer.py,sha256=-qcNSkY6i7ur2LXkM7H1jTKuuFbDuXbjTq42o3vjeQ8,595
|
|
32
32
|
judgeval/scorers/api_scorer.py,sha256=jPBQUBs_T3Xq33QoIbIXDzUaXinz56qeDfo96dfdX0g,2036
|
|
@@ -41,10 +41,10 @@ judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py,sha256=WUeFy
|
|
|
41
41
|
judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py,sha256=ciiFBQQC4UDsk9qou9OiKbAR31s82eRUY1ZTt1gdM-0,407
|
|
42
42
|
judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=ucYOI6ztAjfoYmcgTDzN8u5RrehlVqrkeLEfss9b1fk,441
|
|
43
43
|
judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=V3RdrWhnR_vLBrtWw7QbgN9K_A-Och7-v9I2fN4z8gY,506
|
|
44
|
-
judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=
|
|
45
|
-
judgeval/tracer/__init__.py,sha256=
|
|
46
|
-
judgeval/tracer/constants.py,sha256=
|
|
47
|
-
judgeval/tracer/keys.py,sha256=
|
|
44
|
+
judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=WhSkPs8tWyQ_cS-y-VTzrKAPlizKp-6zi_DmfgW4AgM,10773
|
|
45
|
+
judgeval/tracer/__init__.py,sha256=E1rcegwIQyqEIvM4yfPLm2M0B6e27dhPrJPhMotBjtw,37432
|
|
46
|
+
judgeval/tracer/constants.py,sha256=tLR5ClDaNlNg_MAv2XRdk62uQW4KyBnWaNbG_YYblTc,55
|
|
47
|
+
judgeval/tracer/keys.py,sha256=mYBo_X6-rC9xfiI-WpjHlO7rUtcMORtQXCQyO1F3Ycc,2387
|
|
48
48
|
judgeval/tracer/managers.py,sha256=NEkovnK8Qaod1U5msT0_hyHUqo9C2uFro2IzNlC8jCo,5071
|
|
49
49
|
judgeval/tracer/utils.py,sha256=xWha5iwC733wCf2HKbNqzxOPS1ovO1OymWIUFLz-UpQ,537
|
|
50
50
|
judgeval/tracer/exporters/__init__.py,sha256=3WDXC28iY5gYMM5s7ejmy7P-DVDQ_iIuzwovZxUKJXg,1295
|
|
@@ -57,21 +57,22 @@ judgeval/tracer/llm/constants.py,sha256=IWa3CMes8wIt_UG7jrGEOztg2sHz54fdOMWIOOr-
|
|
|
57
57
|
judgeval/tracer/llm/providers.py,sha256=VAimkmChOOjhC1cUv-0iG8pa5PhOw1HIOyt3zrIrbcM,628
|
|
58
58
|
judgeval/tracer/llm/llm_anthropic/__init__.py,sha256=HG0gIlTgaRt-Y0u1ERPQ19pUgb4YHkTh7tZQPeyR4oM,80
|
|
59
59
|
judgeval/tracer/llm/llm_anthropic/config.py,sha256=ICfKODPQvZsRxpK4xWQ-YE79pmWJTmY2wryddxpNdpM,153
|
|
60
|
-
judgeval/tracer/llm/llm_anthropic/messages.py,sha256=
|
|
61
|
-
judgeval/tracer/llm/llm_anthropic/messages_stream.py,sha256=
|
|
60
|
+
judgeval/tracer/llm/llm_anthropic/messages.py,sha256=T7dApxJCsOWEpquYSZICACwTioZG3ZcxHdJjvF04T2E,15474
|
|
61
|
+
judgeval/tracer/llm/llm_anthropic/messages_stream.py,sha256=DKlZZnfK_yv_tEMwF2XxvsjgUjOFI3c5JUMQwERNV7k,12188
|
|
62
62
|
judgeval/tracer/llm/llm_anthropic/wrapper.py,sha256=JILcyC4NvjXZSqlFoZp-VB-JsCYZkQPMFEYaB4AysrA,1849
|
|
63
63
|
judgeval/tracer/llm/llm_google/__init__.py,sha256=otBZETsAfVZjtZaN5N36Ln0kw-I9jVB4tFGrV6novHo,74
|
|
64
64
|
judgeval/tracer/llm/llm_google/config.py,sha256=S3yCAE9oHbXjLVYiz5mGD16yIgXMBBUu5UN4lBjoCNQ,162
|
|
65
|
-
judgeval/tracer/llm/llm_google/generate_content.py,sha256=
|
|
65
|
+
judgeval/tracer/llm/llm_google/generate_content.py,sha256=w1rIh1cTBYnkfBQTL4qHntwsKfBcSrf2VSS2y-BOMRU,4030
|
|
66
66
|
judgeval/tracer/llm/llm_google/wrapper.py,sha256=jqaMXGoM9dlPBbCFadMI5EqFrNHzBt0h9VkNn7KPVLk,901
|
|
67
67
|
judgeval/tracer/llm/llm_openai/__init__.py,sha256=CyzwhY0-zmqWKlEno7JPBcvO7G_hI8dp6-_5_KEzFqg,74
|
|
68
|
-
judgeval/tracer/llm/llm_openai/beta_chat_completions.py,sha256=
|
|
69
|
-
judgeval/tracer/llm/llm_openai/chat_completions.py,sha256=
|
|
68
|
+
judgeval/tracer/llm/llm_openai/beta_chat_completions.py,sha256=IXw-Gu-WUxQ-gaBUIe-aAKOn1Pakn_RFl0b1C_1toP8,7326
|
|
69
|
+
judgeval/tracer/llm/llm_openai/chat_completions.py,sha256=HaZEZMla-VbacJTyOYeVNNXsRSSqE2di36NIpyzGBVY,17394
|
|
70
70
|
judgeval/tracer/llm/llm_openai/config.py,sha256=NE0ixKhd4WVeAVjY8jNTncuKYH6R4MQDLPmcCsd3zWY,144
|
|
71
|
-
judgeval/tracer/llm/llm_openai/responses.py,sha256=
|
|
71
|
+
judgeval/tracer/llm/llm_openai/responses.py,sha256=5le9rmUtLeVnz10DKS5ajkVcVCHQL3hi1b89pihmoL4,17872
|
|
72
|
+
judgeval/tracer/llm/llm_openai/utils.py,sha256=H32OYE9JeN2Hmyf8qRjqz2ky9QCjSV7KAEKXNAtDWpE,832
|
|
72
73
|
judgeval/tracer/llm/llm_openai/wrapper.py,sha256=Z5Ndib228yd1pXEQ4xIu7_CJHxpW_t0ofZAC6FLc5eU,2055
|
|
73
74
|
judgeval/tracer/llm/llm_together/__init__.py,sha256=MEnsF77IgFD4h73hNCMpo-9a1PHHdm-OxPlOalXOMac,78
|
|
74
|
-
judgeval/tracer/llm/llm_together/chat_completions.py,sha256=
|
|
75
|
+
judgeval/tracer/llm/llm_together/chat_completions.py,sha256=RySsK3tqG0NpJHPlVQ705bXxIfseSQUhvIoS-sz4rOg,14380
|
|
75
76
|
judgeval/tracer/llm/llm_together/config.py,sha256=jCJY0KQcHJZZJk2vq038GKIDUMusqgvRjQ0B6OV5uEc,150
|
|
76
77
|
judgeval/tracer/llm/llm_together/wrapper.py,sha256=HFqy_MabQeSq8oj2diZhEuk1SDt_hDfk5MFdPn9MFhg,1733
|
|
77
78
|
judgeval/tracer/processors/__init__.py,sha256=BdOOPOD1RfMI5YHW76DNPKR07EAev-JxoolZ3KaXNNU,7100
|
|
@@ -84,10 +85,10 @@ judgeval/trainer/trainable_model.py,sha256=T-Sioi_sXtfYlcu3lE0cd60PHs8DrYaZ-Kxb4
|
|
|
84
85
|
judgeval/trainer/trainer.py,sha256=twLEHNaomelTg6ZYG6veI9OpB3wzhPCtPVQMTnDZWx4,2626
|
|
85
86
|
judgeval/utils/async_utils.py,sha256=AF1xdu8Ao5GyhFvfaLOaKJHn1RISyXZ4U70UZe9zfBA,1083
|
|
86
87
|
judgeval/utils/file_utils.py,sha256=vq-n5WZEZjVbZ5S9QTkW8nSH6Pvw-Jx0ttsQ1t0wnPQ,3140
|
|
87
|
-
judgeval/utils/guards.py,sha256=
|
|
88
|
+
judgeval/utils/guards.py,sha256=_DaKZxvjD10J97Ze2paHhbCiV2MpDz3FZQmNwaL5k0w,945
|
|
88
89
|
judgeval/utils/meta.py,sha256=RAqZuvOlymqMwFoS0joBW_r65lcN9bY8BpNYHoytKps,773
|
|
89
90
|
judgeval/utils/project.py,sha256=kGpYmp6QGTD6h-GjQ-ovT7kBmGnyb99MWDJmRGFQHOg,527
|
|
90
|
-
judgeval/utils/serialize.py,sha256=
|
|
91
|
+
judgeval/utils/serialize.py,sha256=WbforbVFGINuk68T2YtWhj-ECMC6rWol3g5dxz9nsm8,6265
|
|
91
92
|
judgeval/utils/testing.py,sha256=m5Nexv65tmfSj1XvAPK5Ear7aJ7w5xjDtZN0tLZ_RBk,2939
|
|
92
93
|
judgeval/utils/url.py,sha256=Shf0v3XcbaWpL0m1eGJEEO_z4TsQCnDB2Rl25OTUmiI,195
|
|
93
94
|
judgeval/utils/version_check.py,sha256=se4Ft8rjcl5u7fHMxSGQpka844V2AcZpOYl6StLWTio,1081
|
|
@@ -104,8 +105,8 @@ judgeval/utils/wrappers/mutable_wrap_async.py,sha256=stHISOUCGFUJXY8seXmxUo4ZpMF
|
|
|
104
105
|
judgeval/utils/wrappers/mutable_wrap_sync.py,sha256=t5jygAQ1vqhy8s1GfiLeYygYgaLTgfoYASN47U5JiPs,2888
|
|
105
106
|
judgeval/utils/wrappers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
106
107
|
judgeval/utils/wrappers/utils.py,sha256=j18vaa6JWDw2s3nQy1z5PfV_9Xxio-bVARaHG_0XyL0,1228
|
|
107
|
-
judgeval-0.
|
|
108
|
-
judgeval-0.
|
|
109
|
-
judgeval-0.
|
|
110
|
-
judgeval-0.
|
|
111
|
-
judgeval-0.
|
|
108
|
+
judgeval-0.22.0.dist-info/METADATA,sha256=xPrltYImsdj9e2CAxKmwop-VJCf9ZPXzgKi83qkUm8E,11483
|
|
109
|
+
judgeval-0.22.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
110
|
+
judgeval-0.22.0.dist-info/entry_points.txt,sha256=-eoeD-oDLn4A7MSgeBS9Akwanf3_0r0cgEleBcIOjg0,46
|
|
111
|
+
judgeval-0.22.0.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
|
|
112
|
+
judgeval-0.22.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|