judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- judgeval/__init__.py +173 -10
- judgeval/api/__init__.py +523 -0
- judgeval/api/api_types.py +413 -0
- judgeval/cli.py +112 -0
- judgeval/constants.py +7 -30
- judgeval/data/__init__.py +1 -3
- judgeval/data/evaluation_run.py +125 -0
- judgeval/data/example.py +14 -40
- judgeval/data/judgment_types.py +396 -146
- judgeval/data/result.py +11 -18
- judgeval/data/scorer_data.py +3 -26
- judgeval/data/scripts/openapi_transform.py +5 -5
- judgeval/data/trace.py +115 -194
- judgeval/dataset/__init__.py +335 -0
- judgeval/env.py +55 -0
- judgeval/evaluation/__init__.py +346 -0
- judgeval/exceptions.py +28 -0
- judgeval/integrations/langgraph/__init__.py +13 -0
- judgeval/integrations/openlit/__init__.py +51 -0
- judgeval/judges/__init__.py +2 -2
- judgeval/judges/litellm_judge.py +77 -16
- judgeval/judges/together_judge.py +88 -17
- judgeval/judges/utils.py +7 -20
- judgeval/judgment_attribute_keys.py +55 -0
- judgeval/{common/logger.py → logger.py} +24 -8
- judgeval/prompt/__init__.py +330 -0
- judgeval/scorers/__init__.py +11 -11
- judgeval/scorers/agent_scorer.py +15 -19
- judgeval/scorers/api_scorer.py +21 -23
- judgeval/scorers/base_scorer.py +54 -36
- judgeval/scorers/example_scorer.py +1 -3
- judgeval/scorers/judgeval_scorers/api_scorers/__init__.py +2 -24
- judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py +2 -10
- judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py +2 -2
- judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py +2 -10
- judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py +2 -14
- judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py +171 -59
- judgeval/scorers/score.py +64 -47
- judgeval/scorers/utils.py +2 -107
- judgeval/tracer/__init__.py +1111 -2
- judgeval/tracer/constants.py +1 -0
- judgeval/tracer/exporters/__init__.py +40 -0
- judgeval/tracer/exporters/s3.py +119 -0
- judgeval/tracer/exporters/store.py +59 -0
- judgeval/tracer/exporters/utils.py +32 -0
- judgeval/tracer/keys.py +63 -0
- judgeval/tracer/llm/__init__.py +7 -0
- judgeval/tracer/llm/config.py +78 -0
- judgeval/tracer/llm/constants.py +9 -0
- judgeval/tracer/llm/llm_anthropic/__init__.py +3 -0
- judgeval/tracer/llm/llm_anthropic/config.py +6 -0
- judgeval/tracer/llm/llm_anthropic/messages.py +452 -0
- judgeval/tracer/llm/llm_anthropic/messages_stream.py +322 -0
- judgeval/tracer/llm/llm_anthropic/wrapper.py +59 -0
- judgeval/tracer/llm/llm_google/__init__.py +3 -0
- judgeval/tracer/llm/llm_google/config.py +6 -0
- judgeval/tracer/llm/llm_google/generate_content.py +127 -0
- judgeval/tracer/llm/llm_google/wrapper.py +30 -0
- judgeval/tracer/llm/llm_openai/__init__.py +3 -0
- judgeval/tracer/llm/llm_openai/beta_chat_completions.py +216 -0
- judgeval/tracer/llm/llm_openai/chat_completions.py +501 -0
- judgeval/tracer/llm/llm_openai/config.py +6 -0
- judgeval/tracer/llm/llm_openai/responses.py +506 -0
- judgeval/tracer/llm/llm_openai/utils.py +42 -0
- judgeval/tracer/llm/llm_openai/wrapper.py +63 -0
- judgeval/tracer/llm/llm_together/__init__.py +3 -0
- judgeval/tracer/llm/llm_together/chat_completions.py +406 -0
- judgeval/tracer/llm/llm_together/config.py +6 -0
- judgeval/tracer/llm/llm_together/wrapper.py +52 -0
- judgeval/tracer/llm/providers.py +19 -0
- judgeval/tracer/managers.py +167 -0
- judgeval/tracer/processors/__init__.py +220 -0
- judgeval/tracer/utils.py +19 -0
- judgeval/trainer/__init__.py +14 -0
- judgeval/trainer/base_trainer.py +122 -0
- judgeval/trainer/config.py +123 -0
- judgeval/trainer/console.py +144 -0
- judgeval/trainer/fireworks_trainer.py +392 -0
- judgeval/trainer/trainable_model.py +252 -0
- judgeval/trainer/trainer.py +70 -0
- judgeval/utils/async_utils.py +39 -0
- judgeval/utils/decorators/__init__.py +0 -0
- judgeval/utils/decorators/dont_throw.py +37 -0
- judgeval/utils/decorators/use_once.py +13 -0
- judgeval/utils/file_utils.py +74 -28
- judgeval/utils/guards.py +36 -0
- judgeval/utils/meta.py +27 -0
- judgeval/utils/project.py +15 -0
- judgeval/utils/serialize.py +253 -0
- judgeval/utils/testing.py +70 -0
- judgeval/utils/url.py +10 -0
- judgeval/{version_check.py → utils/version_check.py} +5 -3
- judgeval/utils/wrappers/README.md +3 -0
- judgeval/utils/wrappers/__init__.py +15 -0
- judgeval/utils/wrappers/immutable_wrap_async.py +74 -0
- judgeval/utils/wrappers/immutable_wrap_async_iterator.py +84 -0
- judgeval/utils/wrappers/immutable_wrap_sync.py +66 -0
- judgeval/utils/wrappers/immutable_wrap_sync_iterator.py +84 -0
- judgeval/utils/wrappers/mutable_wrap_async.py +67 -0
- judgeval/utils/wrappers/mutable_wrap_sync.py +67 -0
- judgeval/utils/wrappers/py.typed +0 -0
- judgeval/utils/wrappers/utils.py +35 -0
- judgeval/v1/__init__.py +88 -0
- judgeval/v1/data/__init__.py +7 -0
- judgeval/v1/data/example.py +44 -0
- judgeval/v1/data/scorer_data.py +42 -0
- judgeval/v1/data/scoring_result.py +44 -0
- judgeval/v1/datasets/__init__.py +6 -0
- judgeval/v1/datasets/dataset.py +214 -0
- judgeval/v1/datasets/dataset_factory.py +94 -0
- judgeval/v1/evaluation/__init__.py +6 -0
- judgeval/v1/evaluation/evaluation.py +182 -0
- judgeval/v1/evaluation/evaluation_factory.py +17 -0
- judgeval/v1/instrumentation/__init__.py +6 -0
- judgeval/v1/instrumentation/llm/__init__.py +7 -0
- judgeval/v1/instrumentation/llm/config.py +78 -0
- judgeval/v1/instrumentation/llm/constants.py +11 -0
- judgeval/v1/instrumentation/llm/llm_anthropic/__init__.py +5 -0
- judgeval/v1/instrumentation/llm/llm_anthropic/config.py +6 -0
- judgeval/v1/instrumentation/llm/llm_anthropic/messages.py +414 -0
- judgeval/v1/instrumentation/llm/llm_anthropic/messages_stream.py +307 -0
- judgeval/v1/instrumentation/llm/llm_anthropic/wrapper.py +61 -0
- judgeval/v1/instrumentation/llm/llm_google/__init__.py +5 -0
- judgeval/v1/instrumentation/llm/llm_google/config.py +6 -0
- judgeval/v1/instrumentation/llm/llm_google/generate_content.py +121 -0
- judgeval/v1/instrumentation/llm/llm_google/wrapper.py +30 -0
- judgeval/v1/instrumentation/llm/llm_openai/__init__.py +5 -0
- judgeval/v1/instrumentation/llm/llm_openai/beta_chat_completions.py +212 -0
- judgeval/v1/instrumentation/llm/llm_openai/chat_completions.py +477 -0
- judgeval/v1/instrumentation/llm/llm_openai/config.py +6 -0
- judgeval/v1/instrumentation/llm/llm_openai/responses.py +472 -0
- judgeval/v1/instrumentation/llm/llm_openai/utils.py +41 -0
- judgeval/v1/instrumentation/llm/llm_openai/wrapper.py +63 -0
- judgeval/v1/instrumentation/llm/llm_together/__init__.py +5 -0
- judgeval/v1/instrumentation/llm/llm_together/chat_completions.py +382 -0
- judgeval/v1/instrumentation/llm/llm_together/config.py +6 -0
- judgeval/v1/instrumentation/llm/llm_together/wrapper.py +57 -0
- judgeval/v1/instrumentation/llm/providers.py +19 -0
- judgeval/v1/integrations/claude_agent_sdk/__init__.py +119 -0
- judgeval/v1/integrations/claude_agent_sdk/wrapper.py +564 -0
- judgeval/v1/integrations/langgraph/__init__.py +13 -0
- judgeval/v1/integrations/openlit/__init__.py +47 -0
- judgeval/v1/internal/api/__init__.py +525 -0
- judgeval/v1/internal/api/api_types.py +413 -0
- judgeval/v1/prompts/__init__.py +6 -0
- judgeval/v1/prompts/prompt.py +29 -0
- judgeval/v1/prompts/prompt_factory.py +189 -0
- judgeval/v1/py.typed +0 -0
- judgeval/v1/scorers/__init__.py +6 -0
- judgeval/v1/scorers/api_scorer.py +82 -0
- judgeval/v1/scorers/base_scorer.py +17 -0
- judgeval/v1/scorers/built_in/__init__.py +17 -0
- judgeval/v1/scorers/built_in/answer_correctness.py +28 -0
- judgeval/v1/scorers/built_in/answer_relevancy.py +28 -0
- judgeval/v1/scorers/built_in/built_in_factory.py +26 -0
- judgeval/v1/scorers/built_in/faithfulness.py +28 -0
- judgeval/v1/scorers/built_in/instruction_adherence.py +28 -0
- judgeval/v1/scorers/custom_scorer/__init__.py +6 -0
- judgeval/v1/scorers/custom_scorer/custom_scorer.py +50 -0
- judgeval/v1/scorers/custom_scorer/custom_scorer_factory.py +16 -0
- judgeval/v1/scorers/prompt_scorer/__init__.py +6 -0
- judgeval/v1/scorers/prompt_scorer/prompt_scorer.py +86 -0
- judgeval/v1/scorers/prompt_scorer/prompt_scorer_factory.py +85 -0
- judgeval/v1/scorers/scorers_factory.py +49 -0
- judgeval/v1/tracer/__init__.py +7 -0
- judgeval/v1/tracer/base_tracer.py +520 -0
- judgeval/v1/tracer/exporters/__init__.py +14 -0
- judgeval/v1/tracer/exporters/in_memory_span_exporter.py +25 -0
- judgeval/v1/tracer/exporters/judgment_span_exporter.py +42 -0
- judgeval/v1/tracer/exporters/noop_span_exporter.py +19 -0
- judgeval/v1/tracer/exporters/span_store.py +50 -0
- judgeval/v1/tracer/judgment_tracer_provider.py +70 -0
- judgeval/v1/tracer/processors/__init__.py +6 -0
- judgeval/v1/tracer/processors/_lifecycles/__init__.py +28 -0
- judgeval/v1/tracer/processors/_lifecycles/agent_id_processor.py +53 -0
- judgeval/v1/tracer/processors/_lifecycles/context_keys.py +11 -0
- judgeval/v1/tracer/processors/_lifecycles/customer_id_processor.py +29 -0
- judgeval/v1/tracer/processors/_lifecycles/registry.py +18 -0
- judgeval/v1/tracer/processors/judgment_span_processor.py +165 -0
- judgeval/v1/tracer/processors/noop_span_processor.py +42 -0
- judgeval/v1/tracer/tracer.py +67 -0
- judgeval/v1/tracer/tracer_factory.py +38 -0
- judgeval/v1/trainers/__init__.py +5 -0
- judgeval/v1/trainers/base_trainer.py +62 -0
- judgeval/v1/trainers/config.py +123 -0
- judgeval/v1/trainers/console.py +144 -0
- judgeval/v1/trainers/fireworks_trainer.py +392 -0
- judgeval/v1/trainers/trainable_model.py +252 -0
- judgeval/v1/trainers/trainers_factory.py +37 -0
- judgeval/v1/utils.py +18 -0
- judgeval/version.py +5 -0
- judgeval/warnings.py +4 -0
- judgeval-0.23.0.dist-info/METADATA +266 -0
- judgeval-0.23.0.dist-info/RECORD +201 -0
- judgeval-0.23.0.dist-info/entry_points.txt +2 -0
- judgeval/clients.py +0 -34
- judgeval/common/__init__.py +0 -13
- judgeval/common/api/__init__.py +0 -3
- judgeval/common/api/api.py +0 -352
- judgeval/common/api/constants.py +0 -165
- judgeval/common/exceptions.py +0 -27
- judgeval/common/storage/__init__.py +0 -6
- judgeval/common/storage/s3_storage.py +0 -98
- judgeval/common/tracer/__init__.py +0 -31
- judgeval/common/tracer/constants.py +0 -22
- judgeval/common/tracer/core.py +0 -1916
- judgeval/common/tracer/otel_exporter.py +0 -108
- judgeval/common/tracer/otel_span_processor.py +0 -234
- judgeval/common/tracer/span_processor.py +0 -37
- judgeval/common/tracer/span_transformer.py +0 -211
- judgeval/common/tracer/trace_manager.py +0 -92
- judgeval/common/utils.py +0 -940
- judgeval/data/datasets/__init__.py +0 -4
- judgeval/data/datasets/dataset.py +0 -341
- judgeval/data/datasets/eval_dataset_client.py +0 -214
- judgeval/data/tool.py +0 -5
- judgeval/data/trace_run.py +0 -37
- judgeval/evaluation_run.py +0 -75
- judgeval/integrations/langgraph.py +0 -843
- judgeval/judges/mixture_of_judges.py +0 -286
- judgeval/judgment_client.py +0 -369
- judgeval/rules.py +0 -521
- judgeval/run_evaluation.py +0 -684
- judgeval/scorers/judgeval_scorers/api_scorers/derailment_scorer.py +0 -14
- judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py +0 -52
- judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py +0 -28
- judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py +0 -20
- judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py +0 -27
- judgeval/utils/alerts.py +0 -93
- judgeval/utils/requests.py +0 -50
- judgeval-0.1.0.dist-info/METADATA +0 -202
- judgeval-0.1.0.dist-info/RECORD +0 -73
- {judgeval-0.1.0.dist-info → judgeval-0.23.0.dist-info}/WHEEL +0 -0
- {judgeval-0.1.0.dist-info → judgeval-0.23.0.dist-info}/licenses/LICENSE.md +0 -0
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
judgeval/__init__.py,sha256=eRR4MYz2YruHB1pkPVblQLubqk2bLxyct_wlNVt6RRw,6648
|
|
2
|
+
judgeval/cli.py,sha256=WV3GE5eIfC1gXE32ymd6psCaRbwBfhTJnkjCZu_1qgc,3735
|
|
3
|
+
judgeval/constants.py,sha256=JZZJ1MqzZZDVk-5PRPRbmLnM8mXI-RDL5vxa1JFuscs,3408
|
|
4
|
+
judgeval/env.py,sha256=ys0tPuR-ofYC7bm716lCF6zuCRPFU8-W3Tn4QN_DueI,1961
|
|
5
|
+
judgeval/exceptions.py,sha256=WMD_7E9U3SN5uakWYxbrPiAmqn7WKw17dQBTg59LL64,669
|
|
6
|
+
judgeval/judgment_attribute_keys.py,sha256=ko6zM4_2aErXu-LN6NExrrW5HUZjecCNSVsyrPee2cs,2456
|
|
7
|
+
judgeval/logger.py,sha256=8XjpF-92_vDL6Is3YZRBdEPxkynuRBY9QO_O0Efi6H0,1961
|
|
8
|
+
judgeval/version.py,sha256=GAGpb-jQr_yTED9mQSDGKIa_cM-bS2_CnOCg4eFoQ_I,74
|
|
9
|
+
judgeval/warnings.py,sha256=LbGte14ppiFjrkp-JJYueZ40NWFvMkWRvPXr6r-fUWw,73
|
|
10
|
+
judgeval/api/__init__.py,sha256=4Pg0Vk9HxA9O6-XDqKliRjtCZMuIPtpXXAQPJzRIdoQ,15322
|
|
11
|
+
judgeval/api/api_types.py,sha256=Jyflzz7ilWrMpaWlNmKNuHvS-6FSXel1JKy7j0Zb-_4,10173
|
|
12
|
+
judgeval/data/__init__.py,sha256=1tU0EN0ThIfQ1fad5I3dKxAfTcZ5U8cvTLcQ6qLVLU0,407
|
|
13
|
+
judgeval/data/evaluation_run.py,sha256=O41p99wNAuCAf6lsLNKzkZ6W-kL9LlzCYxVls7IcKkA,4727
|
|
14
|
+
judgeval/data/example.py,sha256=eGJpF-lyUH734Cg90B7WtU9f8iKoS3VFGeV6R-GVCCc,1039
|
|
15
|
+
judgeval/data/judgment_types.py,sha256=eJmVVzorMpYpbft9_RYDKf2IYrHEJl_rIqd71pv_PLE,18998
|
|
16
|
+
judgeval/data/result.py,sha256=XufFGSAkBDfevPUmzSgsR9HEqytISkM0U5HkhJmsjpY,2102
|
|
17
|
+
judgeval/data/scorer_data.py,sha256=HeP15ZgftFTJCF8JmDJCLWXRnZJIaGDJCzl7Hg6gWwE,2006
|
|
18
|
+
judgeval/data/trace.py,sha256=zSiR3o6xt8Z46XA3M9fJBtViF0BsPO6yKp9jxdscOSc,3881
|
|
19
|
+
judgeval/data/scripts/fix_default_factory.py,sha256=lvp2JwYZqz-XpD9LZNa3mANZVP-jJSZoNzolI6JWERM,591
|
|
20
|
+
judgeval/data/scripts/openapi_transform.py,sha256=Sm04JClzyP1ga8KA3gkIdsae8Hlx-XU7-x0gHCQYOhg,3877
|
|
21
|
+
judgeval/dataset/__init__.py,sha256=YhfsHA9pUgqQSWs-_QSpNkgOu8sxBsq6Kn3ccbeA_Qg,11162
|
|
22
|
+
judgeval/evaluation/__init__.py,sha256=lZN4krKrY5wGALmf1Rsa3mICLS4tPEtA522hLwKSfYM,13131
|
|
23
|
+
judgeval/integrations/langgraph/__init__.py,sha256=HwXmtDxaO75Kn4KPErnMb6Ne6FcpRxV_SCYVuwFsve0,332
|
|
24
|
+
judgeval/integrations/openlit/__init__.py,sha256=BOCdyFjBrJ2GLE0lsmSXgY7_fEWczeuFQmH9-aRAKuA,1619
|
|
25
|
+
judgeval/judges/__init__.py,sha256=e7JnTc1TG_SwqydDHTXHIP0EBazQxt-ydMQG7ghSU5A,228
|
|
26
|
+
judgeval/judges/base_judge.py,sha256=_dz0qWsKRxzXxpRY9l6mrxTRYPSF2FE4ZXkrzhZ4gbY,986
|
|
27
|
+
judgeval/judges/litellm_judge.py,sha256=5vEF0IUo7HVWnOF2ww-DMke8Xkarnz32B_qbgKjc0-I,4182
|
|
28
|
+
judgeval/judges/together_judge.py,sha256=GzwlXZJzle8hT-vWKmq39JyIeanJqJfHDOkrksUbzk0,4398
|
|
29
|
+
judgeval/judges/utils.py,sha256=ITbYwvjU3o9-FIAReFvxh24yJrx9LV3l9BnSBgKUpxg,2068
|
|
30
|
+
judgeval/prompt/__init__.py,sha256=Qgrd8u4WniaOjbRAoEFEeMnTmaqIGx5ZGX_U85iqhs0,11010
|
|
31
|
+
judgeval/scorers/__init__.py,sha256=pomKzEy4YNFyygYp8vbS3co8iB5CMstRkQwdUgi1u4g,744
|
|
32
|
+
judgeval/scorers/agent_scorer.py,sha256=-qcNSkY6i7ur2LXkM7H1jTKuuFbDuXbjTq42o3vjeQ8,595
|
|
33
|
+
judgeval/scorers/api_scorer.py,sha256=jPBQUBs_T3Xq33QoIbIXDzUaXinz56qeDfo96dfdX0g,2036
|
|
34
|
+
judgeval/scorers/base_scorer.py,sha256=hsMuqdW8QtW5n9JzruXyaZC7im2K2sSmz1RDkbMisJ4,2702
|
|
35
|
+
judgeval/scorers/example_scorer.py,sha256=o_BGUztJXjnKnuOqIa9T4PXe0wPoWg63FyH518N1LxA,561
|
|
36
|
+
judgeval/scorers/exceptions.py,sha256=ACDHK5-TWiF3NTk-wycaedpbrdobm-CvvC1JA_iP-Mk,179
|
|
37
|
+
judgeval/scorers/score.py,sha256=xquM59SCtNeuAsrBsHFgBQk3CHp4-bms4oFs24xfcU0,7176
|
|
38
|
+
judgeval/scorers/utils.py,sha256=dDxPKVjKa1lsMXNhZ8-aJFG3qk1usAH1JnKeC3vBQbU,304
|
|
39
|
+
judgeval/scorers/judgeval_scorers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
40
|
+
judgeval/scorers/judgeval_scorers/api_scorers/__init__.py,sha256=wrq7y9I30GZbwDXIrSh81KRO_-j7i-1DjwX5Hc3PScI,728
|
|
41
|
+
judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py,sha256=WUeFyWdr1Wc8dh-aQ1nrK-mbd9W0MT4VyzLT5CbJ2-Q,450
|
|
42
|
+
judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py,sha256=ciiFBQQC4UDsk9qou9OiKbAR31s82eRUY1ZTt1gdM-0,407
|
|
43
|
+
judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=ucYOI6ztAjfoYmcgTDzN8u5RrehlVqrkeLEfss9b1fk,441
|
|
44
|
+
judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=V3RdrWhnR_vLBrtWw7QbgN9K_A-Och7-v9I2fN4z8gY,506
|
|
45
|
+
judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=WhSkPs8tWyQ_cS-y-VTzrKAPlizKp-6zi_DmfgW4AgM,10773
|
|
46
|
+
judgeval/tracer/__init__.py,sha256=pxPTipSuuKFOzT0hrM7USTguKu0V-39lfdAloxtNoAM,39296
|
|
47
|
+
judgeval/tracer/constants.py,sha256=tLR5ClDaNlNg_MAv2XRdk62uQW4KyBnWaNbG_YYblTc,55
|
|
48
|
+
judgeval/tracer/keys.py,sha256=mYBo_X6-rC9xfiI-WpjHlO7rUtcMORtQXCQyO1F3Ycc,2387
|
|
49
|
+
judgeval/tracer/managers.py,sha256=JiUjX_evToxcuogKVcE6qpJkSvYOxAXCU4_z_hWXJOw,5199
|
|
50
|
+
judgeval/tracer/utils.py,sha256=xWha5iwC733wCf2HKbNqzxOPS1ovO1OymWIUFLz-UpQ,537
|
|
51
|
+
judgeval/tracer/exporters/__init__.py,sha256=3WDXC28iY5gYMM5s7ejmy7P-DVDQ_iIuzwovZxUKJXg,1295
|
|
52
|
+
judgeval/tracer/exporters/s3.py,sha256=N9gmw17cnR0VkfAQQkLsNj5BksgNRETThR5qYhWRjP4,4360
|
|
53
|
+
judgeval/tracer/exporters/store.py,sha256=pA_KINcm0amO0WEDYmMFU05SSsMOgJ5ogIRaevSX1sk,1885
|
|
54
|
+
judgeval/tracer/exporters/utils.py,sha256=JRcoSQuEHxMDJbXfyrUIfA2SHBVkZM82h4bTbYGxkNw,1154
|
|
55
|
+
judgeval/tracer/llm/__init__.py,sha256=ENxApieKSktYrIviofXWP9GU0WnhBm0Q9mlGe_m_gMY,139
|
|
56
|
+
judgeval/tracer/llm/config.py,sha256=J8-bTL82bgDqdTJPN-Px3Epvoa9FG7L-X329kitwBTc,2525
|
|
57
|
+
judgeval/tracer/llm/constants.py,sha256=IWa3CMes8wIt_UG7jrGEOztg2sHz54fdOMWIOOr-dz8,172
|
|
58
|
+
judgeval/tracer/llm/providers.py,sha256=VAimkmChOOjhC1cUv-0iG8pa5PhOw1HIOyt3zrIrbcM,628
|
|
59
|
+
judgeval/tracer/llm/llm_anthropic/__init__.py,sha256=HG0gIlTgaRt-Y0u1ERPQ19pUgb4YHkTh7tZQPeyR4oM,80
|
|
60
|
+
judgeval/tracer/llm/llm_anthropic/config.py,sha256=ICfKODPQvZsRxpK4xWQ-YE79pmWJTmY2wryddxpNdpM,153
|
|
61
|
+
judgeval/tracer/llm/llm_anthropic/messages.py,sha256=T7dApxJCsOWEpquYSZICACwTioZG3ZcxHdJjvF04T2E,15474
|
|
62
|
+
judgeval/tracer/llm/llm_anthropic/messages_stream.py,sha256=DKlZZnfK_yv_tEMwF2XxvsjgUjOFI3c5JUMQwERNV7k,12188
|
|
63
|
+
judgeval/tracer/llm/llm_anthropic/wrapper.py,sha256=JILcyC4NvjXZSqlFoZp-VB-JsCYZkQPMFEYaB4AysrA,1849
|
|
64
|
+
judgeval/tracer/llm/llm_google/__init__.py,sha256=otBZETsAfVZjtZaN5N36Ln0kw-I9jVB4tFGrV6novHo,74
|
|
65
|
+
judgeval/tracer/llm/llm_google/config.py,sha256=S3yCAE9oHbXjLVYiz5mGD16yIgXMBBUu5UN4lBjoCNQ,162
|
|
66
|
+
judgeval/tracer/llm/llm_google/generate_content.py,sha256=w1rIh1cTBYnkfBQTL4qHntwsKfBcSrf2VSS2y-BOMRU,4030
|
|
67
|
+
judgeval/tracer/llm/llm_google/wrapper.py,sha256=jqaMXGoM9dlPBbCFadMI5EqFrNHzBt0h9VkNn7KPVLk,901
|
|
68
|
+
judgeval/tracer/llm/llm_openai/__init__.py,sha256=CyzwhY0-zmqWKlEno7JPBcvO7G_hI8dp6-_5_KEzFqg,74
|
|
69
|
+
judgeval/tracer/llm/llm_openai/beta_chat_completions.py,sha256=IXw-Gu-WUxQ-gaBUIe-aAKOn1Pakn_RFl0b1C_1toP8,7326
|
|
70
|
+
judgeval/tracer/llm/llm_openai/chat_completions.py,sha256=U086NgaaLFiyvAYrgJncC-obaaSbG2r_3ehquNlVTDQ,17637
|
|
71
|
+
judgeval/tracer/llm/llm_openai/config.py,sha256=NE0ixKhd4WVeAVjY8jNTncuKYH6R4MQDLPmcCsd3zWY,144
|
|
72
|
+
judgeval/tracer/llm/llm_openai/responses.py,sha256=CCGYz35gn3jJOYE2anyR49OR2XhSDwy3dEsISbzMO8Q,18137
|
|
73
|
+
judgeval/tracer/llm/llm_openai/utils.py,sha256=fpy9war8dyke25qHxGW2Yo028RA4Siq0RBLA4G63yUw,1480
|
|
74
|
+
judgeval/tracer/llm/llm_openai/wrapper.py,sha256=Z5Ndib228yd1pXEQ4xIu7_CJHxpW_t0ofZAC6FLc5eU,2055
|
|
75
|
+
judgeval/tracer/llm/llm_together/__init__.py,sha256=MEnsF77IgFD4h73hNCMpo-9a1PHHdm-OxPlOalXOMac,78
|
|
76
|
+
judgeval/tracer/llm/llm_together/chat_completions.py,sha256=RySsK3tqG0NpJHPlVQ705bXxIfseSQUhvIoS-sz4rOg,14380
|
|
77
|
+
judgeval/tracer/llm/llm_together/config.py,sha256=jCJY0KQcHJZZJk2vq038GKIDUMusqgvRjQ0B6OV5uEc,150
|
|
78
|
+
judgeval/tracer/llm/llm_together/wrapper.py,sha256=HFqy_MabQeSq8oj2diZhEuk1SDt_hDfk5MFdPn9MFhg,1733
|
|
79
|
+
judgeval/tracer/processors/__init__.py,sha256=BdOOPOD1RfMI5YHW76DNPKR07EAev-JxoolZ3KaXNNU,7100
|
|
80
|
+
judgeval/trainer/__init__.py,sha256=nJo913vFdss3E_PR-M1OUjznS0SYgNZ-MP-Y_6Mj5PA,437
|
|
81
|
+
judgeval/trainer/base_trainer.py,sha256=Lxm6OxJpifonLKofNIRG3TU7n_jZWQZ0I_f_jwtb_WU,4018
|
|
82
|
+
judgeval/trainer/config.py,sha256=jL8VsjMTPNeRPfhsrOaOiRLjx9BYBc_XnyDw8ncvRqk,4073
|
|
83
|
+
judgeval/trainer/console.py,sha256=SvokkFEU-K1vLV4Rd1m6YJJ7HyYwTr4Azdzwx_JPZUY,4351
|
|
84
|
+
judgeval/trainer/fireworks_trainer.py,sha256=5BvjOWeRA6DOWez861ewWgRYyqpnMG22Sxj2Hqz4L50,15933
|
|
85
|
+
judgeval/trainer/trainable_model.py,sha256=96XI2_y1M8LiT7xZJ9Ydu7BiRn_iAGj6MI5n_orD2iA,9433
|
|
86
|
+
judgeval/trainer/trainer.py,sha256=twLEHNaomelTg6ZYG6veI9OpB3wzhPCtPVQMTnDZWx4,2626
|
|
87
|
+
judgeval/utils/async_utils.py,sha256=AF1xdu8Ao5GyhFvfaLOaKJHn1RISyXZ4U70UZe9zfBA,1083
|
|
88
|
+
judgeval/utils/file_utils.py,sha256=vq-n5WZEZjVbZ5S9QTkW8nSH6Pvw-Jx0ttsQ1t0wnPQ,3140
|
|
89
|
+
judgeval/utils/guards.py,sha256=_DaKZxvjD10J97Ze2paHhbCiV2MpDz3FZQmNwaL5k0w,945
|
|
90
|
+
judgeval/utils/meta.py,sha256=RAqZuvOlymqMwFoS0joBW_r65lcN9bY8BpNYHoytKps,773
|
|
91
|
+
judgeval/utils/project.py,sha256=kGpYmp6QGTD6h-GjQ-ovT7kBmGnyb99MWDJmRGFQHOg,527
|
|
92
|
+
judgeval/utils/serialize.py,sha256=WbforbVFGINuk68T2YtWhj-ECMC6rWol3g5dxz9nsm8,6265
|
|
93
|
+
judgeval/utils/testing.py,sha256=m5Nexv65tmfSj1XvAPK5Ear7aJ7w5xjDtZN0tLZ_RBk,2939
|
|
94
|
+
judgeval/utils/url.py,sha256=Shf0v3XcbaWpL0m1eGJEEO_z4TsQCnDB2Rl25OTUmiI,195
|
|
95
|
+
judgeval/utils/version_check.py,sha256=se4Ft8rjcl5u7fHMxSGQpka844V2AcZpOYl6StLWTio,1081
|
|
96
|
+
judgeval/utils/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
97
|
+
judgeval/utils/decorators/dont_throw.py,sha256=B6N-4RhHxMn55TqMImVGG82BVzh3yOPmcrmSflT7f4I,989
|
|
98
|
+
judgeval/utils/decorators/use_once.py,sha256=8mgj5VK9v08VOOWX2Bstc0CezNsOVUKMIv7N2R83E8s,288
|
|
99
|
+
judgeval/utils/wrappers/README.md,sha256=-Jyagu6NPH92ty8pTMbzRLVJZzufULrjxcyohXgsGMc,76
|
|
100
|
+
judgeval/utils/wrappers/__init__.py,sha256=iAcpjCOkYqoe6z2utrS_3yZLmdQPD1Y64MMefai8h0Y,546
|
|
101
|
+
judgeval/utils/wrappers/immutable_wrap_async.py,sha256=a0LWyEa235tPpfjN3W0A516_GWDL13uhumMVzsMpgW4,1909
|
|
102
|
+
judgeval/utils/wrappers/immutable_wrap_async_iterator.py,sha256=hfVxBoFE6m6I0g0KcSLJXyfVv6pfZuoJuHyLJHtLjjg,2268
|
|
103
|
+
judgeval/utils/wrappers/immutable_wrap_sync.py,sha256=_gOUaPK4Le-pifWCZOH4lDvY-cLfYoC0fy7DTNIG-0A,1823
|
|
104
|
+
judgeval/utils/wrappers/immutable_wrap_sync_iterator.py,sha256=aDC4HpLp4l9A3aFLS0cTCkien-xGgQRU04F7P1pJ6w8,2229
|
|
105
|
+
judgeval/utils/wrappers/mutable_wrap_async.py,sha256=stHISOUCGFUJXY8seXmxUo4ZpMF4LErSBIz0HlWR7Bo,2941
|
|
106
|
+
judgeval/utils/wrappers/mutable_wrap_sync.py,sha256=t5jygAQ1vqhy8s1GfiLeYygYgaLTgfoYASN47U5JiPs,2888
|
|
107
|
+
judgeval/utils/wrappers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
108
|
+
judgeval/utils/wrappers/utils.py,sha256=j18vaa6JWDw2s3nQy1z5PfV_9Xxio-bVARaHG_0XyL0,1228
|
|
109
|
+
judgeval/v1/__init__.py,sha256=xfEruFmFiMC0gqp3IcJynOMqo5h8nqiZLgCLZpjsEJU,2322
|
|
110
|
+
judgeval/v1/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
111
|
+
judgeval/v1/utils.py,sha256=Vw34Ouzmos6yuTyrV2jHY9Ym9jbBApOOZ3--7bYPt7g,632
|
|
112
|
+
judgeval/v1/data/__init__.py,sha256=hPTyLqn5XViwOnYmNL17YUkyVmSxx42mv4JDxmrQcTk,245
|
|
113
|
+
judgeval/v1/data/example.py,sha256=Q4N-GHQeEAIfVd4_GMaV6MCZHCzRYZOPPtsBvBh3RW0,1355
|
|
114
|
+
judgeval/v1/data/scorer_data.py,sha256=mYip_rz7HfG12sd_Z61ZKUGzZpjpZHgbJBRh5h3eUlA,1391
|
|
115
|
+
judgeval/v1/data/scoring_result.py,sha256=u4PJzCcCEl-I8L0GZtkIp_LZIV0mZrliWxWximSHz6Q,1540
|
|
116
|
+
judgeval/v1/datasets/__init__.py,sha256=Q0R1n993aKLp6sy5wmYDYfrU4wsWjBsBuPmx3EGXncQ,218
|
|
117
|
+
judgeval/v1/datasets/dataset.py,sha256=LP3dBYanJLWODdVVH0eGF9CKPR2hkoLXsUw0SigjeJ8,6935
|
|
118
|
+
judgeval/v1/datasets/dataset_factory.py,sha256=bI6ibn37gdQicuLW1lRXGsq5xsCcCDsYVKK6j8Zk_yQ,3202
|
|
119
|
+
judgeval/v1/evaluation/__init__.py,sha256=5zN14e_Ukmx53Q-BnR1KzV3GbhBrS372R62SAJudR9U,212
|
|
120
|
+
judgeval/v1/evaluation/evaluation.py,sha256=k2Jk6o5LYrUy8ye6UktXJBUemBGHQ440MU7xp93RQUI,6738
|
|
121
|
+
judgeval/v1/evaluation/evaluation_factory.py,sha256=kp-YSAVjdpTFMouCg3aMpDakHpLmvYRHduBphfCDJ1Y,392
|
|
122
|
+
judgeval/v1/instrumentation/__init__.py,sha256=7kmEJrqVsKsFqOAXNywRLzQZ6irmtvEGZhb7zLBa0E0,85
|
|
123
|
+
judgeval/v1/instrumentation/llm/__init__.py,sha256=ENxApieKSktYrIviofXWP9GU0WnhBm0Q9mlGe_m_gMY,139
|
|
124
|
+
judgeval/v1/instrumentation/llm/config.py,sha256=CyxJ90GuxcqVUqkucuA6F2fWhgfVNJX1tIO2-DXirx0,2572
|
|
125
|
+
judgeval/v1/instrumentation/llm/constants.py,sha256=1BaM6XEzrCu-DCUeUXxu-p0K5XyWqBXeFQ-Ks_9mp0g,208
|
|
126
|
+
judgeval/v1/instrumentation/llm/providers.py,sha256=KdPNi0G69aG1OfIlXy7PoGLx10Z8n0jqVLKNFZi7RQI,676
|
|
127
|
+
judgeval/v1/instrumentation/llm/llm_anthropic/__init__.py,sha256=iqv1FpU6pldnPRLlhaq5_H1CQbNnquzB3mcWqpTJ68w,116
|
|
128
|
+
judgeval/v1/instrumentation/llm/llm_anthropic/config.py,sha256=ICfKODPQvZsRxpK4xWQ-YE79pmWJTmY2wryddxpNdpM,153
|
|
129
|
+
judgeval/v1/instrumentation/llm/llm_anthropic/messages.py,sha256=8JAwMnYxeUlYLP2zpPIIPmQ2e6zveYBundrDdxKJ-Mc,14823
|
|
130
|
+
judgeval/v1/instrumentation/llm/llm_anthropic/messages_stream.py,sha256=JiB3e1GkAnITuzW6pi7i0xyiBk_TCSqQuDkrIvpLNGA,11766
|
|
131
|
+
judgeval/v1/instrumentation/llm/llm_anthropic/wrapper.py,sha256=vW5UeS0uabF-Rn7hVrpPSiUcM2hYV4naZYcOXgnO8D4,1918
|
|
132
|
+
judgeval/v1/instrumentation/llm/llm_google/__init__.py,sha256=-LUKGeKGFXTveWLSODbM_42hdFbyLEWbapfh6LT-j1s,110
|
|
133
|
+
judgeval/v1/instrumentation/llm/llm_google/config.py,sha256=S3yCAE9oHbXjLVYiz5mGD16yIgXMBBUu5UN4lBjoCNQ,162
|
|
134
|
+
judgeval/v1/instrumentation/llm/llm_google/generate_content.py,sha256=UP6rCCJoa68nZBD0NqbJBlscSJuRSBhK39qs1FxzQD0,3916
|
|
135
|
+
judgeval/v1/instrumentation/llm/llm_google/wrapper.py,sha256=z7EECZ8c3nEKFTkXaCCBFPog3OeH__NaiHQulGBpS1g,936
|
|
136
|
+
judgeval/v1/instrumentation/llm/llm_openai/__init__.py,sha256=fUnFiufoikJkA6hkLV0UUljBIt42lR5Xy0wx5sIXfMg,110
|
|
137
|
+
judgeval/v1/instrumentation/llm/llm_openai/beta_chat_completions.py,sha256=PO0mdOFjKsLe8wa4d9sJ4JNVkK9fuw9CswX0scYb0VE,7229
|
|
138
|
+
judgeval/v1/instrumentation/llm/llm_openai/chat_completions.py,sha256=G3z-O7FvmhsOg3RcjfTUwQCmO_H1F9LecfYJuDmTD7g,17224
|
|
139
|
+
judgeval/v1/instrumentation/llm/llm_openai/config.py,sha256=NE0ixKhd4WVeAVjY8jNTncuKYH6R4MQDLPmcCsd3zWY,144
|
|
140
|
+
judgeval/v1/instrumentation/llm/llm_openai/responses.py,sha256=FS81UOe63ea0-eKEfh2nY958nvLWyDDG3RhTsNe2hqA,17382
|
|
141
|
+
judgeval/v1/instrumentation/llm/llm_openai/utils.py,sha256=l_81Om6IYel7fxYSwGYxFwHncZhjKM4oucXIAGg6kFE,1357
|
|
142
|
+
judgeval/v1/instrumentation/llm/llm_openai/wrapper.py,sha256=cY8Y3H4mLm6OT1UleHVArKFpuQdBf0iI_JJJLOz07Yo,2130
|
|
143
|
+
judgeval/v1/instrumentation/llm/llm_together/__init__.py,sha256=HsZB7HdnbWANuLyH-eUXze-gzoV2oJaBzEMzvcRilu4,114
|
|
144
|
+
judgeval/v1/instrumentation/llm/llm_together/chat_completions.py,sha256=oFKBb8wc_NJyYFxg-XRnEo5ZrpqK_FWhjNYH8WpgO0M,14021
|
|
145
|
+
judgeval/v1/instrumentation/llm/llm_together/config.py,sha256=jCJY0KQcHJZZJk2vq038GKIDUMusqgvRjQ0B6OV5uEc,150
|
|
146
|
+
judgeval/v1/instrumentation/llm/llm_together/wrapper.py,sha256=fSB3gt9S-NWnzaTJUiRtiblbojg6sRftGlIeENv8RK8,1801
|
|
147
|
+
judgeval/v1/integrations/claude_agent_sdk/__init__.py,sha256=5ozUauaDwqqzHGpJifcAEwTLeL83_YEYJtkiU0JGg3E,4552
|
|
148
|
+
judgeval/v1/integrations/claude_agent_sdk/wrapper.py,sha256=san7IhyTHmX0LpEQhsHiQWeidcJDKUJ2LxmnRm0Vg7Q,21496
|
|
149
|
+
judgeval/v1/integrations/langgraph/__init__.py,sha256=HwXmtDxaO75Kn4KPErnMb6Ne6FcpRxV_SCYVuwFsve0,332
|
|
150
|
+
judgeval/v1/integrations/openlit/__init__.py,sha256=Eq-RRKQiDkzNslPjXy_EdpkHTF5Kwd1XnGluVMxOZuk,1400
|
|
151
|
+
judgeval/v1/internal/api/__init__.py,sha256=8NzoAyfYfeCj04F9nB5hCYmi4ar-A6dD9T4ZPUwWxrU,16174
|
|
152
|
+
judgeval/v1/internal/api/api_types.py,sha256=EIbnT8NqEh5Rt6kF6r0hu394vUdBga2lq1Awl47GXJE,10176
|
|
153
|
+
judgeval/v1/prompts/__init__.py,sha256=_en7nUIQzhicHAmLMNT9bcRVhdUiaIizWofhok-V8iI,182
|
|
154
|
+
judgeval/v1/prompts/prompt.py,sha256=AW5XFPQIDEAFHH2uqbkUUN5zvpgjFJJ7MPFAJIXhr4M,835
|
|
155
|
+
judgeval/v1/prompts/prompt_factory.py,sha256=S5PmJYyep_zRNvcwSaxCFn7e1HHT7_rYEIZwJSQc-jY,5689
|
|
156
|
+
judgeval/v1/scorers/__init__.py,sha256=GIzpQmcbsDsIvJs4CO2baata1QzwYAdk1N-GdywJ_9Y,198
|
|
157
|
+
judgeval/v1/scorers/api_scorer.py,sha256=m6KXtrEejKlY125EwYSsun31fIMABiYXAOqwuOqYfZA,2360
|
|
158
|
+
judgeval/v1/scorers/base_scorer.py,sha256=nhem2oU4_OkS5-IHVVepe1lsNalu_VzdBWyIP3SP5XE,325
|
|
159
|
+
judgeval/v1/scorers/scorers_factory.py,sha256=Hk2iyx5yhdki4fVwBHyNxouJZAoaazTyh9M-02qrSgA,1169
|
|
160
|
+
judgeval/v1/scorers/built_in/__init__.py,sha256=H6CS3q_k5G42lQgswCDYbyolTelE1cvgTsPFYJPGmsY,616
|
|
161
|
+
judgeval/v1/scorers/built_in/answer_correctness.py,sha256=PqRsLOFAPy84N0o8ow59bH0CrL-fabSZ9dSWothF3yA,824
|
|
162
|
+
judgeval/v1/scorers/built_in/answer_relevancy.py,sha256=oQGyNSkUNSlcIPa41a6vy31ZKxcE3ONE29hp6IlvufQ,797
|
|
163
|
+
judgeval/v1/scorers/built_in/built_in_factory.py,sha256=ptdSEya2-TJWArMDJvXU-ECsE1Xmd7uXvdJfO17Ra8A,995
|
|
164
|
+
judgeval/v1/scorers/built_in/faithfulness.py,sha256=YwT4J4B8e6vW5gfJfpSzUaet20xpzyKiCVyG36PfSMY,786
|
|
165
|
+
judgeval/v1/scorers/built_in/instruction_adherence.py,sha256=5vOi6zXewryK5L_1hHNnhQbVRTqusH5ftCcoXs36k0M,817
|
|
166
|
+
judgeval/v1/scorers/custom_scorer/__init__.py,sha256=LqIDe6Lftv1DiX-sY6maN7fKUjFiXWNj6Gdk66O3gyk,248
|
|
167
|
+
judgeval/v1/scorers/custom_scorer/custom_scorer.py,sha256=mLQ-ZKzjGX78aky2F44ipQCBjQ6Fw2hlkUxabZ8wirU,1257
|
|
168
|
+
judgeval/v1/scorers/custom_scorer/custom_scorer_factory.py,sha256=BBCGDuxRPff1Xv_nJFqJrxss__Dk5p1jOhd5aYOqBiI,404
|
|
169
|
+
judgeval/v1/scorers/prompt_scorer/__init__.py,sha256=lXu2aIRKzgPqBzai9paMdx8sGiUu9k6-GeTS15OJNGk,248
|
|
170
|
+
judgeval/v1/scorers/prompt_scorer/prompt_scorer.py,sha256=spl3ISlPkp9DP6feavuWjK18yAa0SjxWPfzO9KfDpA8,2494
|
|
171
|
+
judgeval/v1/scorers/prompt_scorer/prompt_scorer_factory.py,sha256=Rg7hPltO1zk0SbkkgA4d5vOhk1S_FKm_r7nrKZ64HVU,2821
|
|
172
|
+
judgeval/v1/tracer/__init__.py,sha256=EkmERuaDI3PTjwy9L5HB7UCfuEJ-tqSzDGTR_NG5Y1M,248
|
|
173
|
+
judgeval/v1/tracer/base_tracer.py,sha256=vp97UTm-dSi9-HpBw2OAVq53G9bN2uHGdB2DRTBPCGQ,18327
|
|
174
|
+
judgeval/v1/tracer/judgment_tracer_provider.py,sha256=i4lKcbc209-6XN-ymUR0ugRLTEw8R0_l6bwbDybmzvs,2358
|
|
175
|
+
judgeval/v1/tracer/tracer.py,sha256=H71bvnn39YtA5VuQAYydst6y_1FncZuhdCyEd1HW9fA,2327
|
|
176
|
+
judgeval/v1/tracer/tracer_factory.py,sha256=0MRtbvoLpb8aQuDqOeobxu-dTVKrLgQBCHUFjZqMVwQ,1075
|
|
177
|
+
judgeval/v1/tracer/exporters/__init__.py,sha256=5dafftPbJlzBQQUSat44I8oZF0XLo1D_Ulsf2A9D3QY,492
|
|
178
|
+
judgeval/v1/tracer/exporters/in_memory_span_exporter.py,sha256=qc7PCnxdxt0Sx2GyysOHe5AMmG2p3sDhEoalAVjvSaI,685
|
|
179
|
+
judgeval/v1/tracer/exporters/judgment_span_exporter.py,sha256=C9P4tZR5Qtlhrv-Xbjpi05way9Irx54wJnniAFXHJ9Q,1313
|
|
180
|
+
judgeval/v1/tracer/exporters/noop_span_exporter.py,sha256=q_xwtfnULw34ENYDWOiN4OsbOq_l0teqh2GCWB0oiUI,492
|
|
181
|
+
judgeval/v1/tracer/exporters/span_store.py,sha256=YmzDX_uCnRn_Y6J-oXSX0hDAvhI1lSCW5PJjBjBBVQE,1503
|
|
182
|
+
judgeval/v1/tracer/processors/__init__.py,sha256=5XqsiixgxFG7X4AzmFLYmiGYCV7FtrW5JWzjyPAEPzg,278
|
|
183
|
+
judgeval/v1/tracer/processors/judgment_span_processor.py,sha256=-pYbJRV_wBEJ_zmFlx7gzp0wSuzqieaDWh3SYDlq0zk,5421
|
|
184
|
+
judgeval/v1/tracer/processors/noop_span_processor.py,sha256=zx5ZcfRFqWkPh_ClLO5qXnYLxUW6aE1NEVSHx65-5mk,1129
|
|
185
|
+
judgeval/v1/tracer/processors/_lifecycles/__init__.py,sha256=tiZ2__ccbRsm4eJCNPbtL-_KYZjJY8pwkkGyqdUmQOA,737
|
|
186
|
+
judgeval/v1/tracer/processors/_lifecycles/agent_id_processor.py,sha256=KW0tmMpVXiOCvOxqsKjbR3HRA7DsW7djHDL38Jj1W70,1861
|
|
187
|
+
judgeval/v1/tracer/processors/_lifecycles/context_keys.py,sha256=yRV9GfPELOr6b2KlVOW4VhVnJJ0oxzW6WnV5Q0gLc7s,495
|
|
188
|
+
judgeval/v1/tracer/processors/_lifecycles/customer_id_processor.py,sha256=afWFAYvfJG89EdD4K_ENaA1sXU0urzkd-tQp-9fh2fk,954
|
|
189
|
+
judgeval/v1/tracer/processors/_lifecycles/registry.py,sha256=24AiMjKts65TlE2vF6xYNtgmsycZ9gpnoylHVOzIdsw,431
|
|
190
|
+
judgeval/v1/trainers/__init__.py,sha256=VUmT_y-zwWmAmtv_IjUNiNwr4BjdZK1-LNor7OkcSE0,133
|
|
191
|
+
judgeval/v1/trainers/base_trainer.py,sha256=sMe1_lFIPhLtr9ck5WwbYiiPD-2w2fHbI-hJ1KOnt-Y,1835
|
|
192
|
+
judgeval/v1/trainers/config.py,sha256=jL8VsjMTPNeRPfhsrOaOiRLjx9BYBc_XnyDw8ncvRqk,4073
|
|
193
|
+
judgeval/v1/trainers/console.py,sha256=SvokkFEU-K1vLV4Rd1m6YJJ7HyYwTr4Azdzwx_JPZUY,4351
|
|
194
|
+
judgeval/v1/trainers/fireworks_trainer.py,sha256=Jx41Kbgfwl8-pR8u3WGZvrrmviFWwzfvgwlVxoD5Sn0,16110
|
|
195
|
+
judgeval/v1/trainers/trainable_model.py,sha256=96XI2_y1M8LiT7xZJ9Ydu7BiRn_iAGj6MI5n_orD2iA,9433
|
|
196
|
+
judgeval/v1/trainers/trainers_factory.py,sha256=a6eABP_FZiP0x3jMBNOKlM-EKcDiQvQS4d6YZ0PedQU,946
|
|
197
|
+
judgeval-0.23.0.dist-info/METADATA,sha256=Len3xf4OuZOkR0CmMbo4KLfDVTxmV7kkwecHB3ucfBw,11598
|
|
198
|
+
judgeval-0.23.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
199
|
+
judgeval-0.23.0.dist-info/entry_points.txt,sha256=-eoeD-oDLn4A7MSgeBS9Akwanf3_0r0cgEleBcIOjg0,46
|
|
200
|
+
judgeval-0.23.0.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
|
|
201
|
+
judgeval-0.23.0.dist-info/RECORD,,
|
judgeval/clients.py
DELETED
|
@@ -1,34 +0,0 @@
|
|
|
1
|
-
import os
|
|
2
|
-
from dotenv import load_dotenv
|
|
3
|
-
from openai import OpenAI
|
|
4
|
-
from typing import Optional
|
|
5
|
-
from together import Together, AsyncTogether
|
|
6
|
-
|
|
7
|
-
PATH_TO_DOTENV = os.path.join(os.path.dirname(__file__), ".env")
|
|
8
|
-
load_dotenv(dotenv_path=PATH_TO_DOTENV)
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
# Initialize optional OpenAI client
|
|
12
|
-
client: Optional["OpenAI"] = None
|
|
13
|
-
if os.getenv("OPENAI_API_KEY"):
|
|
14
|
-
try:
|
|
15
|
-
from openai import OpenAI
|
|
16
|
-
|
|
17
|
-
client = OpenAI()
|
|
18
|
-
except ImportError:
|
|
19
|
-
# openai package not installed
|
|
20
|
-
pass
|
|
21
|
-
|
|
22
|
-
# Initialize optional Together clients
|
|
23
|
-
together_client: Optional["Together"] = None
|
|
24
|
-
async_together_client: Optional["AsyncTogether"] = None
|
|
25
|
-
|
|
26
|
-
# Only initialize Together clients if API key is available
|
|
27
|
-
|
|
28
|
-
together_api_key = os.getenv("TOGETHERAI_API_KEY") or os.getenv("TOGETHER_API_KEY")
|
|
29
|
-
if together_api_key:
|
|
30
|
-
try:
|
|
31
|
-
together_client = Together(api_key=together_api_key)
|
|
32
|
-
async_together_client = AsyncTogether(api_key=together_api_key)
|
|
33
|
-
except Exception:
|
|
34
|
-
pass
|
judgeval/common/__init__.py
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
from judgeval.common.utils import (
|
|
2
|
-
get_chat_completion,
|
|
3
|
-
aget_chat_completion,
|
|
4
|
-
get_completion_multiple_models,
|
|
5
|
-
aget_completion_multiple_models,
|
|
6
|
-
)
|
|
7
|
-
|
|
8
|
-
__all__ = [
|
|
9
|
-
"get_chat_completion",
|
|
10
|
-
"aget_chat_completion",
|
|
11
|
-
"get_completion_multiple_models",
|
|
12
|
-
"aget_completion_multiple_models",
|
|
13
|
-
]
|
judgeval/common/api/__init__.py
DELETED
judgeval/common/api/api.py
DELETED
|
@@ -1,352 +0,0 @@
|
|
|
1
|
-
from typing import Literal, List, Dict, Any
|
|
2
|
-
from requests import exceptions
|
|
3
|
-
from judgeval.common.api.constants import (
|
|
4
|
-
JUDGMENT_TRACES_FETCH_API_URL,
|
|
5
|
-
JUDGMENT_TRACES_UPSERT_API_URL,
|
|
6
|
-
JUDGMENT_TRACES_DELETE_API_URL,
|
|
7
|
-
JUDGMENT_TRACES_SPANS_BATCH_API_URL,
|
|
8
|
-
JUDGMENT_TRACES_EVALUATION_RUNS_BATCH_API_URL,
|
|
9
|
-
JUDGMENT_DATASETS_PUSH_API_URL,
|
|
10
|
-
JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL,
|
|
11
|
-
JUDGMENT_DATASETS_PULL_API_URL,
|
|
12
|
-
JUDGMENT_DATASETS_DELETE_API_URL,
|
|
13
|
-
JUDGMENT_DATASETS_PROJECT_STATS_API_URL,
|
|
14
|
-
JUDGMENT_PROJECT_DELETE_API_URL,
|
|
15
|
-
JUDGMENT_PROJECT_CREATE_API_URL,
|
|
16
|
-
JUDGMENT_EVAL_API_URL,
|
|
17
|
-
JUDGMENT_TRACE_EVAL_API_URL,
|
|
18
|
-
JUDGMENT_EVAL_LOG_API_URL,
|
|
19
|
-
JUDGMENT_EVAL_FETCH_API_URL,
|
|
20
|
-
JUDGMENT_EVAL_DELETE_API_URL,
|
|
21
|
-
JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL,
|
|
22
|
-
JUDGMENT_GET_EVAL_STATUS_API_URL,
|
|
23
|
-
JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL,
|
|
24
|
-
JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL,
|
|
25
|
-
JUDGMENT_SCORER_SAVE_API_URL,
|
|
26
|
-
JUDGMENT_SCORER_FETCH_API_URL,
|
|
27
|
-
JUDGMENT_SCORER_EXISTS_API_URL,
|
|
28
|
-
)
|
|
29
|
-
from judgeval.common.api.constants import (
|
|
30
|
-
TraceFetchPayload,
|
|
31
|
-
TraceDeletePayload,
|
|
32
|
-
SpansBatchPayload,
|
|
33
|
-
EvaluationEntryResponse,
|
|
34
|
-
EvaluationRunsBatchPayload,
|
|
35
|
-
DatasetPushPayload,
|
|
36
|
-
DatasetAppendPayload,
|
|
37
|
-
DatasetPullPayload,
|
|
38
|
-
DatasetDeletePayload,
|
|
39
|
-
DatasetStatsPayload,
|
|
40
|
-
ProjectCreatePayload,
|
|
41
|
-
ProjectDeletePayload,
|
|
42
|
-
EvalRunRequestBody,
|
|
43
|
-
DeleteEvalRunRequestBody,
|
|
44
|
-
EvalLogPayload,
|
|
45
|
-
EvalStatusPayload,
|
|
46
|
-
CheckExperimentTypePayload,
|
|
47
|
-
EvalRunNameExistsPayload,
|
|
48
|
-
ScorerSavePayload,
|
|
49
|
-
ScorerFetchPayload,
|
|
50
|
-
ScorerExistsPayload,
|
|
51
|
-
)
|
|
52
|
-
from judgeval.utils.requests import requests
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
class JudgmentAPIException(exceptions.HTTPError):
|
|
56
|
-
"""
|
|
57
|
-
Exception raised when an error occurs while executing a Judgment API request.
|
|
58
|
-
Extends requests.exceptions.HTTPError to provide access to the response object.
|
|
59
|
-
"""
|
|
60
|
-
|
|
61
|
-
def __init__(self, message: str, response=None, request=None):
|
|
62
|
-
super().__init__(message, response=response, request=request)
|
|
63
|
-
self.message = message
|
|
64
|
-
self.response = response
|
|
65
|
-
self.request = request
|
|
66
|
-
|
|
67
|
-
@property
|
|
68
|
-
def status_code(self) -> int | None:
|
|
69
|
-
"""Get the HTTP status code from the response."""
|
|
70
|
-
return self.response.status_code if self.response else None
|
|
71
|
-
|
|
72
|
-
@property
|
|
73
|
-
def response_json(self) -> Dict[str, Any]:
|
|
74
|
-
"""Get the JSON response body."""
|
|
75
|
-
try:
|
|
76
|
-
return self.response.json() if self.response else {}
|
|
77
|
-
except (ValueError, AttributeError):
|
|
78
|
-
return {}
|
|
79
|
-
|
|
80
|
-
@property
|
|
81
|
-
def error_detail(self) -> str:
|
|
82
|
-
"""Get the error detail from the response JSON."""
|
|
83
|
-
return self.response_json.get("detail", "An unknown error occurred.")
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
class JudgmentApiClient:
|
|
87
|
-
def __init__(self, api_key: str, organization_id: str):
|
|
88
|
-
self.api_key = api_key
|
|
89
|
-
self.organization_id = organization_id
|
|
90
|
-
|
|
91
|
-
def _do_request(
|
|
92
|
-
self,
|
|
93
|
-
method: Literal["POST", "PATCH", "GET", "DELETE"],
|
|
94
|
-
url: str,
|
|
95
|
-
payload: Any,
|
|
96
|
-
) -> Any:
|
|
97
|
-
if method == "GET":
|
|
98
|
-
r = requests.request(
|
|
99
|
-
method,
|
|
100
|
-
url,
|
|
101
|
-
params=payload,
|
|
102
|
-
headers=self._headers(),
|
|
103
|
-
**self._request_kwargs(),
|
|
104
|
-
)
|
|
105
|
-
else:
|
|
106
|
-
r = requests.request(
|
|
107
|
-
method,
|
|
108
|
-
url,
|
|
109
|
-
data=self._serialize(payload),
|
|
110
|
-
headers=self._headers(),
|
|
111
|
-
**self._request_kwargs(),
|
|
112
|
-
)
|
|
113
|
-
|
|
114
|
-
try:
|
|
115
|
-
r.raise_for_status()
|
|
116
|
-
except exceptions.HTTPError as e:
|
|
117
|
-
raise JudgmentAPIException(
|
|
118
|
-
f"HTTP {r.status_code}: {r.reason}", response=r, request=e.request
|
|
119
|
-
)
|
|
120
|
-
|
|
121
|
-
return r.json()
|
|
122
|
-
|
|
123
|
-
def send_spans_batch(self, spans: List[Dict[str, Any]]):
|
|
124
|
-
payload: SpansBatchPayload = {
|
|
125
|
-
"spans": spans,
|
|
126
|
-
"organization_id": self.organization_id,
|
|
127
|
-
}
|
|
128
|
-
|
|
129
|
-
return self._do_request("POST", JUDGMENT_TRACES_SPANS_BATCH_API_URL, payload)
|
|
130
|
-
|
|
131
|
-
def send_evaluation_runs_batch(
|
|
132
|
-
self, evaluation_entries: List[EvaluationEntryResponse]
|
|
133
|
-
):
|
|
134
|
-
payload: EvaluationRunsBatchPayload = {
|
|
135
|
-
"organization_id": self.organization_id,
|
|
136
|
-
"evaluation_entries": evaluation_entries,
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
return self._do_request(
|
|
140
|
-
"POST", JUDGMENT_TRACES_EVALUATION_RUNS_BATCH_API_URL, payload
|
|
141
|
-
)
|
|
142
|
-
|
|
143
|
-
def fetch_trace(self, trace_id: str):
|
|
144
|
-
payload: TraceFetchPayload = {"trace_id": trace_id}
|
|
145
|
-
return self._do_request("POST", JUDGMENT_TRACES_FETCH_API_URL, payload)
|
|
146
|
-
|
|
147
|
-
def upsert_trace(self, trace_data: Dict[str, Any]):
|
|
148
|
-
return self._do_request("POST", JUDGMENT_TRACES_UPSERT_API_URL, trace_data)
|
|
149
|
-
|
|
150
|
-
def delete_trace(self, trace_id: str):
|
|
151
|
-
payload: TraceDeletePayload = {"trace_ids": [trace_id]}
|
|
152
|
-
return self._do_request("DELETE", JUDGMENT_TRACES_DELETE_API_URL, payload)
|
|
153
|
-
|
|
154
|
-
def delete_traces(self, trace_ids: List[str]):
|
|
155
|
-
payload: TraceDeletePayload = {"trace_ids": trace_ids}
|
|
156
|
-
return self._do_request("DELETE", JUDGMENT_TRACES_DELETE_API_URL, payload)
|
|
157
|
-
|
|
158
|
-
def delete_project(self, project_name: str):
|
|
159
|
-
payload: ProjectDeletePayload = {"project_name": project_name}
|
|
160
|
-
return self._do_request("DELETE", JUDGMENT_PROJECT_DELETE_API_URL, payload)
|
|
161
|
-
|
|
162
|
-
def create_project(self, project_name: str):
|
|
163
|
-
payload: ProjectCreatePayload = {"project_name": project_name}
|
|
164
|
-
return self._do_request("POST", JUDGMENT_PROJECT_CREATE_API_URL, payload)
|
|
165
|
-
|
|
166
|
-
def run_evaluation(self, evaluation_run: Dict[str, Any]):
|
|
167
|
-
return self._do_request("POST", JUDGMENT_EVAL_API_URL, evaluation_run)
|
|
168
|
-
|
|
169
|
-
def run_trace_evaluation(self, trace_run: Dict[str, Any]):
|
|
170
|
-
return self._do_request("POST", JUDGMENT_TRACE_EVAL_API_URL, trace_run)
|
|
171
|
-
|
|
172
|
-
def log_evaluation_results(
|
|
173
|
-
self, results: List[Dict[str, Any]], run: Dict[str, Any]
|
|
174
|
-
):
|
|
175
|
-
payload: EvalLogPayload = {"results": results, "run": run}
|
|
176
|
-
return self._do_request("POST", JUDGMENT_EVAL_LOG_API_URL, payload)
|
|
177
|
-
|
|
178
|
-
def fetch_evaluation_results(self, project_name: str, eval_name: str):
|
|
179
|
-
payload: EvalRunRequestBody = {
|
|
180
|
-
"project_name": project_name,
|
|
181
|
-
"eval_name": eval_name,
|
|
182
|
-
}
|
|
183
|
-
return self._do_request("POST", JUDGMENT_EVAL_FETCH_API_URL, payload)
|
|
184
|
-
|
|
185
|
-
def delete_evaluation_results(self, project_name: str, eval_names: List[str]):
|
|
186
|
-
payload: DeleteEvalRunRequestBody = {
|
|
187
|
-
"project_name": project_name,
|
|
188
|
-
"eval_names": eval_names,
|
|
189
|
-
"judgment_api_key": self.api_key,
|
|
190
|
-
}
|
|
191
|
-
return self._do_request("POST", JUDGMENT_EVAL_DELETE_API_URL, payload)
|
|
192
|
-
|
|
193
|
-
def add_to_evaluation_queue(self, payload: Dict[str, Any]):
|
|
194
|
-
return self._do_request("POST", JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL, payload)
|
|
195
|
-
|
|
196
|
-
def get_evaluation_status(self, eval_name: str, project_name: str):
|
|
197
|
-
payload: EvalStatusPayload = {
|
|
198
|
-
"eval_name": eval_name,
|
|
199
|
-
"project_name": project_name,
|
|
200
|
-
"judgment_api_key": self.api_key,
|
|
201
|
-
}
|
|
202
|
-
return self._do_request("GET", JUDGMENT_GET_EVAL_STATUS_API_URL, payload)
|
|
203
|
-
|
|
204
|
-
def check_experiment_type(self, eval_name: str, project_name: str, is_trace: bool):
|
|
205
|
-
payload: CheckExperimentTypePayload = {
|
|
206
|
-
"eval_name": eval_name,
|
|
207
|
-
"project_name": project_name,
|
|
208
|
-
"judgment_api_key": self.api_key,
|
|
209
|
-
"is_trace": is_trace,
|
|
210
|
-
}
|
|
211
|
-
return self._do_request("POST", JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL, payload)
|
|
212
|
-
|
|
213
|
-
def check_eval_run_name_exists(self, eval_name: str, project_name: str):
|
|
214
|
-
payload: EvalRunNameExistsPayload = {
|
|
215
|
-
"eval_name": eval_name,
|
|
216
|
-
"project_name": project_name,
|
|
217
|
-
"judgment_api_key": self.api_key,
|
|
218
|
-
}
|
|
219
|
-
return self._do_request("POST", JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL, payload)
|
|
220
|
-
|
|
221
|
-
def save_scorer(self, name: str, prompt: str, options: dict):
|
|
222
|
-
payload: ScorerSavePayload = {
|
|
223
|
-
"name": name,
|
|
224
|
-
"prompt": prompt,
|
|
225
|
-
"options": options,
|
|
226
|
-
}
|
|
227
|
-
try:
|
|
228
|
-
return self._do_request("POST", JUDGMENT_SCORER_SAVE_API_URL, payload)
|
|
229
|
-
except JudgmentAPIException as e:
|
|
230
|
-
if e.status_code == 500:
|
|
231
|
-
raise JudgmentAPIException(
|
|
232
|
-
f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}",
|
|
233
|
-
response=e.response,
|
|
234
|
-
request=e.request,
|
|
235
|
-
)
|
|
236
|
-
raise JudgmentAPIException(
|
|
237
|
-
f"Failed to save classifier scorer: {e.error_detail}",
|
|
238
|
-
response=e.response,
|
|
239
|
-
request=e.request,
|
|
240
|
-
)
|
|
241
|
-
|
|
242
|
-
def fetch_scorer(self, name: str):
|
|
243
|
-
payload: ScorerFetchPayload = {"name": name}
|
|
244
|
-
try:
|
|
245
|
-
return self._do_request("POST", JUDGMENT_SCORER_FETCH_API_URL, payload)
|
|
246
|
-
except JudgmentAPIException as e:
|
|
247
|
-
if e.status_code == 500:
|
|
248
|
-
raise JudgmentAPIException(
|
|
249
|
-
f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}",
|
|
250
|
-
response=e.response,
|
|
251
|
-
request=e.request,
|
|
252
|
-
)
|
|
253
|
-
raise JudgmentAPIException(
|
|
254
|
-
f"Failed to fetch classifier scorer '{name}': {e.error_detail}",
|
|
255
|
-
response=e.response,
|
|
256
|
-
request=e.request,
|
|
257
|
-
)
|
|
258
|
-
|
|
259
|
-
def scorer_exists(self, name: str):
|
|
260
|
-
payload: ScorerExistsPayload = {"name": name}
|
|
261
|
-
try:
|
|
262
|
-
return self._do_request("POST", JUDGMENT_SCORER_EXISTS_API_URL, payload)
|
|
263
|
-
except JudgmentAPIException as e:
|
|
264
|
-
if e.status_code == 500:
|
|
265
|
-
raise JudgmentAPIException(
|
|
266
|
-
f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}",
|
|
267
|
-
response=e.response,
|
|
268
|
-
request=e.request,
|
|
269
|
-
)
|
|
270
|
-
raise JudgmentAPIException(
|
|
271
|
-
f"Failed to check if scorer exists: {e.error_detail}",
|
|
272
|
-
response=e.response,
|
|
273
|
-
request=e.request,
|
|
274
|
-
)
|
|
275
|
-
|
|
276
|
-
def push_dataset(
|
|
277
|
-
self,
|
|
278
|
-
dataset_alias: str,
|
|
279
|
-
project_name: str,
|
|
280
|
-
examples: List[Dict[str, Any]],
|
|
281
|
-
traces: List[Dict[str, Any]],
|
|
282
|
-
overwrite: bool,
|
|
283
|
-
):
|
|
284
|
-
payload: DatasetPushPayload = {
|
|
285
|
-
"dataset_alias": dataset_alias,
|
|
286
|
-
"project_name": project_name,
|
|
287
|
-
"examples": examples,
|
|
288
|
-
"traces": traces,
|
|
289
|
-
"overwrite": overwrite,
|
|
290
|
-
}
|
|
291
|
-
return self._do_request("POST", JUDGMENT_DATASETS_PUSH_API_URL, payload)
|
|
292
|
-
|
|
293
|
-
def append_examples(
|
|
294
|
-
self, dataset_alias: str, project_name: str, examples: List[Dict[str, Any]]
|
|
295
|
-
):
|
|
296
|
-
payload: DatasetAppendPayload = {
|
|
297
|
-
"dataset_alias": dataset_alias,
|
|
298
|
-
"project_name": project_name,
|
|
299
|
-
"examples": examples,
|
|
300
|
-
}
|
|
301
|
-
return self._do_request(
|
|
302
|
-
"POST", JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL, payload
|
|
303
|
-
)
|
|
304
|
-
|
|
305
|
-
def pull_dataset(self, dataset_alias: str, project_name: str):
|
|
306
|
-
payload: DatasetPullPayload = {
|
|
307
|
-
"dataset_alias": dataset_alias,
|
|
308
|
-
"project_name": project_name,
|
|
309
|
-
}
|
|
310
|
-
return self._do_request("POST", JUDGMENT_DATASETS_PULL_API_URL, payload)
|
|
311
|
-
|
|
312
|
-
def delete_dataset(self, dataset_alias: str, project_name: str):
|
|
313
|
-
payload: DatasetDeletePayload = {
|
|
314
|
-
"dataset_alias": dataset_alias,
|
|
315
|
-
"project_name": project_name,
|
|
316
|
-
}
|
|
317
|
-
return self._do_request("POST", JUDGMENT_DATASETS_DELETE_API_URL, payload)
|
|
318
|
-
|
|
319
|
-
def get_project_dataset_stats(self, project_name: str):
|
|
320
|
-
payload: DatasetStatsPayload = {"project_name": project_name}
|
|
321
|
-
return self._do_request(
|
|
322
|
-
"POST", JUDGMENT_DATASETS_PROJECT_STATS_API_URL, payload
|
|
323
|
-
)
|
|
324
|
-
|
|
325
|
-
def _headers(self) -> Dict[str, str]:
|
|
326
|
-
return {
|
|
327
|
-
"Content-Type": "application/json",
|
|
328
|
-
"Authorization": f"Bearer {self.api_key}",
|
|
329
|
-
"X-Organization-Id": self.organization_id,
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
def _request_kwargs(self):
|
|
333
|
-
# NOTE: We may want to configure custom kwargs that different requests may need.
|
|
334
|
-
# For this purpose we can store that as a property of self, and return the appropriate kwargs from this method.
|
|
335
|
-
return {
|
|
336
|
-
"verify": True,
|
|
337
|
-
"timeout": 30,
|
|
338
|
-
}
|
|
339
|
-
|
|
340
|
-
def _serialize(self, data: Any) -> str:
|
|
341
|
-
def fallback_encoder(obj):
|
|
342
|
-
try:
|
|
343
|
-
return repr(obj)
|
|
344
|
-
except Exception:
|
|
345
|
-
try:
|
|
346
|
-
return str(obj)
|
|
347
|
-
except Exception as e:
|
|
348
|
-
return f"<Unserializable object of type {type(obj).__name__}: {e}>"
|
|
349
|
-
|
|
350
|
-
import json
|
|
351
|
-
|
|
352
|
-
return json.dumps(data, default=fallback_encoder)
|