judgeval 0.1.0__py3-none-any.whl → 0.23.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. judgeval/__init__.py +173 -10
  2. judgeval/api/__init__.py +523 -0
  3. judgeval/api/api_types.py +413 -0
  4. judgeval/cli.py +112 -0
  5. judgeval/constants.py +7 -30
  6. judgeval/data/__init__.py +1 -3
  7. judgeval/data/evaluation_run.py +125 -0
  8. judgeval/data/example.py +14 -40
  9. judgeval/data/judgment_types.py +396 -146
  10. judgeval/data/result.py +11 -18
  11. judgeval/data/scorer_data.py +3 -26
  12. judgeval/data/scripts/openapi_transform.py +5 -5
  13. judgeval/data/trace.py +115 -194
  14. judgeval/dataset/__init__.py +335 -0
  15. judgeval/env.py +55 -0
  16. judgeval/evaluation/__init__.py +346 -0
  17. judgeval/exceptions.py +28 -0
  18. judgeval/integrations/langgraph/__init__.py +13 -0
  19. judgeval/integrations/openlit/__init__.py +51 -0
  20. judgeval/judges/__init__.py +2 -2
  21. judgeval/judges/litellm_judge.py +77 -16
  22. judgeval/judges/together_judge.py +88 -17
  23. judgeval/judges/utils.py +7 -20
  24. judgeval/judgment_attribute_keys.py +55 -0
  25. judgeval/{common/logger.py → logger.py} +24 -8
  26. judgeval/prompt/__init__.py +330 -0
  27. judgeval/scorers/__init__.py +11 -11
  28. judgeval/scorers/agent_scorer.py +15 -19
  29. judgeval/scorers/api_scorer.py +21 -23
  30. judgeval/scorers/base_scorer.py +54 -36
  31. judgeval/scorers/example_scorer.py +1 -3
  32. judgeval/scorers/judgeval_scorers/api_scorers/__init__.py +2 -24
  33. judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py +2 -10
  34. judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py +2 -2
  35. judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py +2 -10
  36. judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py +2 -14
  37. judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py +171 -59
  38. judgeval/scorers/score.py +64 -47
  39. judgeval/scorers/utils.py +2 -107
  40. judgeval/tracer/__init__.py +1111 -2
  41. judgeval/tracer/constants.py +1 -0
  42. judgeval/tracer/exporters/__init__.py +40 -0
  43. judgeval/tracer/exporters/s3.py +119 -0
  44. judgeval/tracer/exporters/store.py +59 -0
  45. judgeval/tracer/exporters/utils.py +32 -0
  46. judgeval/tracer/keys.py +63 -0
  47. judgeval/tracer/llm/__init__.py +7 -0
  48. judgeval/tracer/llm/config.py +78 -0
  49. judgeval/tracer/llm/constants.py +9 -0
  50. judgeval/tracer/llm/llm_anthropic/__init__.py +3 -0
  51. judgeval/tracer/llm/llm_anthropic/config.py +6 -0
  52. judgeval/tracer/llm/llm_anthropic/messages.py +452 -0
  53. judgeval/tracer/llm/llm_anthropic/messages_stream.py +322 -0
  54. judgeval/tracer/llm/llm_anthropic/wrapper.py +59 -0
  55. judgeval/tracer/llm/llm_google/__init__.py +3 -0
  56. judgeval/tracer/llm/llm_google/config.py +6 -0
  57. judgeval/tracer/llm/llm_google/generate_content.py +127 -0
  58. judgeval/tracer/llm/llm_google/wrapper.py +30 -0
  59. judgeval/tracer/llm/llm_openai/__init__.py +3 -0
  60. judgeval/tracer/llm/llm_openai/beta_chat_completions.py +216 -0
  61. judgeval/tracer/llm/llm_openai/chat_completions.py +501 -0
  62. judgeval/tracer/llm/llm_openai/config.py +6 -0
  63. judgeval/tracer/llm/llm_openai/responses.py +506 -0
  64. judgeval/tracer/llm/llm_openai/utils.py +42 -0
  65. judgeval/tracer/llm/llm_openai/wrapper.py +63 -0
  66. judgeval/tracer/llm/llm_together/__init__.py +3 -0
  67. judgeval/tracer/llm/llm_together/chat_completions.py +406 -0
  68. judgeval/tracer/llm/llm_together/config.py +6 -0
  69. judgeval/tracer/llm/llm_together/wrapper.py +52 -0
  70. judgeval/tracer/llm/providers.py +19 -0
  71. judgeval/tracer/managers.py +167 -0
  72. judgeval/tracer/processors/__init__.py +220 -0
  73. judgeval/tracer/utils.py +19 -0
  74. judgeval/trainer/__init__.py +14 -0
  75. judgeval/trainer/base_trainer.py +122 -0
  76. judgeval/trainer/config.py +123 -0
  77. judgeval/trainer/console.py +144 -0
  78. judgeval/trainer/fireworks_trainer.py +392 -0
  79. judgeval/trainer/trainable_model.py +252 -0
  80. judgeval/trainer/trainer.py +70 -0
  81. judgeval/utils/async_utils.py +39 -0
  82. judgeval/utils/decorators/__init__.py +0 -0
  83. judgeval/utils/decorators/dont_throw.py +37 -0
  84. judgeval/utils/decorators/use_once.py +13 -0
  85. judgeval/utils/file_utils.py +74 -28
  86. judgeval/utils/guards.py +36 -0
  87. judgeval/utils/meta.py +27 -0
  88. judgeval/utils/project.py +15 -0
  89. judgeval/utils/serialize.py +253 -0
  90. judgeval/utils/testing.py +70 -0
  91. judgeval/utils/url.py +10 -0
  92. judgeval/{version_check.py → utils/version_check.py} +5 -3
  93. judgeval/utils/wrappers/README.md +3 -0
  94. judgeval/utils/wrappers/__init__.py +15 -0
  95. judgeval/utils/wrappers/immutable_wrap_async.py +74 -0
  96. judgeval/utils/wrappers/immutable_wrap_async_iterator.py +84 -0
  97. judgeval/utils/wrappers/immutable_wrap_sync.py +66 -0
  98. judgeval/utils/wrappers/immutable_wrap_sync_iterator.py +84 -0
  99. judgeval/utils/wrappers/mutable_wrap_async.py +67 -0
  100. judgeval/utils/wrappers/mutable_wrap_sync.py +67 -0
  101. judgeval/utils/wrappers/py.typed +0 -0
  102. judgeval/utils/wrappers/utils.py +35 -0
  103. judgeval/v1/__init__.py +88 -0
  104. judgeval/v1/data/__init__.py +7 -0
  105. judgeval/v1/data/example.py +44 -0
  106. judgeval/v1/data/scorer_data.py +42 -0
  107. judgeval/v1/data/scoring_result.py +44 -0
  108. judgeval/v1/datasets/__init__.py +6 -0
  109. judgeval/v1/datasets/dataset.py +214 -0
  110. judgeval/v1/datasets/dataset_factory.py +94 -0
  111. judgeval/v1/evaluation/__init__.py +6 -0
  112. judgeval/v1/evaluation/evaluation.py +182 -0
  113. judgeval/v1/evaluation/evaluation_factory.py +17 -0
  114. judgeval/v1/instrumentation/__init__.py +6 -0
  115. judgeval/v1/instrumentation/llm/__init__.py +7 -0
  116. judgeval/v1/instrumentation/llm/config.py +78 -0
  117. judgeval/v1/instrumentation/llm/constants.py +11 -0
  118. judgeval/v1/instrumentation/llm/llm_anthropic/__init__.py +5 -0
  119. judgeval/v1/instrumentation/llm/llm_anthropic/config.py +6 -0
  120. judgeval/v1/instrumentation/llm/llm_anthropic/messages.py +414 -0
  121. judgeval/v1/instrumentation/llm/llm_anthropic/messages_stream.py +307 -0
  122. judgeval/v1/instrumentation/llm/llm_anthropic/wrapper.py +61 -0
  123. judgeval/v1/instrumentation/llm/llm_google/__init__.py +5 -0
  124. judgeval/v1/instrumentation/llm/llm_google/config.py +6 -0
  125. judgeval/v1/instrumentation/llm/llm_google/generate_content.py +121 -0
  126. judgeval/v1/instrumentation/llm/llm_google/wrapper.py +30 -0
  127. judgeval/v1/instrumentation/llm/llm_openai/__init__.py +5 -0
  128. judgeval/v1/instrumentation/llm/llm_openai/beta_chat_completions.py +212 -0
  129. judgeval/v1/instrumentation/llm/llm_openai/chat_completions.py +477 -0
  130. judgeval/v1/instrumentation/llm/llm_openai/config.py +6 -0
  131. judgeval/v1/instrumentation/llm/llm_openai/responses.py +472 -0
  132. judgeval/v1/instrumentation/llm/llm_openai/utils.py +41 -0
  133. judgeval/v1/instrumentation/llm/llm_openai/wrapper.py +63 -0
  134. judgeval/v1/instrumentation/llm/llm_together/__init__.py +5 -0
  135. judgeval/v1/instrumentation/llm/llm_together/chat_completions.py +382 -0
  136. judgeval/v1/instrumentation/llm/llm_together/config.py +6 -0
  137. judgeval/v1/instrumentation/llm/llm_together/wrapper.py +57 -0
  138. judgeval/v1/instrumentation/llm/providers.py +19 -0
  139. judgeval/v1/integrations/claude_agent_sdk/__init__.py +119 -0
  140. judgeval/v1/integrations/claude_agent_sdk/wrapper.py +564 -0
  141. judgeval/v1/integrations/langgraph/__init__.py +13 -0
  142. judgeval/v1/integrations/openlit/__init__.py +47 -0
  143. judgeval/v1/internal/api/__init__.py +525 -0
  144. judgeval/v1/internal/api/api_types.py +413 -0
  145. judgeval/v1/prompts/__init__.py +6 -0
  146. judgeval/v1/prompts/prompt.py +29 -0
  147. judgeval/v1/prompts/prompt_factory.py +189 -0
  148. judgeval/v1/py.typed +0 -0
  149. judgeval/v1/scorers/__init__.py +6 -0
  150. judgeval/v1/scorers/api_scorer.py +82 -0
  151. judgeval/v1/scorers/base_scorer.py +17 -0
  152. judgeval/v1/scorers/built_in/__init__.py +17 -0
  153. judgeval/v1/scorers/built_in/answer_correctness.py +28 -0
  154. judgeval/v1/scorers/built_in/answer_relevancy.py +28 -0
  155. judgeval/v1/scorers/built_in/built_in_factory.py +26 -0
  156. judgeval/v1/scorers/built_in/faithfulness.py +28 -0
  157. judgeval/v1/scorers/built_in/instruction_adherence.py +28 -0
  158. judgeval/v1/scorers/custom_scorer/__init__.py +6 -0
  159. judgeval/v1/scorers/custom_scorer/custom_scorer.py +50 -0
  160. judgeval/v1/scorers/custom_scorer/custom_scorer_factory.py +16 -0
  161. judgeval/v1/scorers/prompt_scorer/__init__.py +6 -0
  162. judgeval/v1/scorers/prompt_scorer/prompt_scorer.py +86 -0
  163. judgeval/v1/scorers/prompt_scorer/prompt_scorer_factory.py +85 -0
  164. judgeval/v1/scorers/scorers_factory.py +49 -0
  165. judgeval/v1/tracer/__init__.py +7 -0
  166. judgeval/v1/tracer/base_tracer.py +520 -0
  167. judgeval/v1/tracer/exporters/__init__.py +14 -0
  168. judgeval/v1/tracer/exporters/in_memory_span_exporter.py +25 -0
  169. judgeval/v1/tracer/exporters/judgment_span_exporter.py +42 -0
  170. judgeval/v1/tracer/exporters/noop_span_exporter.py +19 -0
  171. judgeval/v1/tracer/exporters/span_store.py +50 -0
  172. judgeval/v1/tracer/judgment_tracer_provider.py +70 -0
  173. judgeval/v1/tracer/processors/__init__.py +6 -0
  174. judgeval/v1/tracer/processors/_lifecycles/__init__.py +28 -0
  175. judgeval/v1/tracer/processors/_lifecycles/agent_id_processor.py +53 -0
  176. judgeval/v1/tracer/processors/_lifecycles/context_keys.py +11 -0
  177. judgeval/v1/tracer/processors/_lifecycles/customer_id_processor.py +29 -0
  178. judgeval/v1/tracer/processors/_lifecycles/registry.py +18 -0
  179. judgeval/v1/tracer/processors/judgment_span_processor.py +165 -0
  180. judgeval/v1/tracer/processors/noop_span_processor.py +42 -0
  181. judgeval/v1/tracer/tracer.py +67 -0
  182. judgeval/v1/tracer/tracer_factory.py +38 -0
  183. judgeval/v1/trainers/__init__.py +5 -0
  184. judgeval/v1/trainers/base_trainer.py +62 -0
  185. judgeval/v1/trainers/config.py +123 -0
  186. judgeval/v1/trainers/console.py +144 -0
  187. judgeval/v1/trainers/fireworks_trainer.py +392 -0
  188. judgeval/v1/trainers/trainable_model.py +252 -0
  189. judgeval/v1/trainers/trainers_factory.py +37 -0
  190. judgeval/v1/utils.py +18 -0
  191. judgeval/version.py +5 -0
  192. judgeval/warnings.py +4 -0
  193. judgeval-0.23.0.dist-info/METADATA +266 -0
  194. judgeval-0.23.0.dist-info/RECORD +201 -0
  195. judgeval-0.23.0.dist-info/entry_points.txt +2 -0
  196. judgeval/clients.py +0 -34
  197. judgeval/common/__init__.py +0 -13
  198. judgeval/common/api/__init__.py +0 -3
  199. judgeval/common/api/api.py +0 -352
  200. judgeval/common/api/constants.py +0 -165
  201. judgeval/common/exceptions.py +0 -27
  202. judgeval/common/storage/__init__.py +0 -6
  203. judgeval/common/storage/s3_storage.py +0 -98
  204. judgeval/common/tracer/__init__.py +0 -31
  205. judgeval/common/tracer/constants.py +0 -22
  206. judgeval/common/tracer/core.py +0 -1916
  207. judgeval/common/tracer/otel_exporter.py +0 -108
  208. judgeval/common/tracer/otel_span_processor.py +0 -234
  209. judgeval/common/tracer/span_processor.py +0 -37
  210. judgeval/common/tracer/span_transformer.py +0 -211
  211. judgeval/common/tracer/trace_manager.py +0 -92
  212. judgeval/common/utils.py +0 -940
  213. judgeval/data/datasets/__init__.py +0 -4
  214. judgeval/data/datasets/dataset.py +0 -341
  215. judgeval/data/datasets/eval_dataset_client.py +0 -214
  216. judgeval/data/tool.py +0 -5
  217. judgeval/data/trace_run.py +0 -37
  218. judgeval/evaluation_run.py +0 -75
  219. judgeval/integrations/langgraph.py +0 -843
  220. judgeval/judges/mixture_of_judges.py +0 -286
  221. judgeval/judgment_client.py +0 -369
  222. judgeval/rules.py +0 -521
  223. judgeval/run_evaluation.py +0 -684
  224. judgeval/scorers/judgeval_scorers/api_scorers/derailment_scorer.py +0 -14
  225. judgeval/scorers/judgeval_scorers/api_scorers/execution_order.py +0 -52
  226. judgeval/scorers/judgeval_scorers/api_scorers/hallucination.py +0 -28
  227. judgeval/scorers/judgeval_scorers/api_scorers/tool_dependency.py +0 -20
  228. judgeval/scorers/judgeval_scorers/api_scorers/tool_order.py +0 -27
  229. judgeval/utils/alerts.py +0 -93
  230. judgeval/utils/requests.py +0 -50
  231. judgeval-0.1.0.dist-info/METADATA +0 -202
  232. judgeval-0.1.0.dist-info/RECORD +0 -73
  233. {judgeval-0.1.0.dist-info → judgeval-0.23.0.dist-info}/WHEEL +0 -0
  234. {judgeval-0.1.0.dist-info → judgeval-0.23.0.dist-info}/licenses/LICENSE.md +0 -0
@@ -0,0 +1,201 @@
1
+ judgeval/__init__.py,sha256=eRR4MYz2YruHB1pkPVblQLubqk2bLxyct_wlNVt6RRw,6648
2
+ judgeval/cli.py,sha256=WV3GE5eIfC1gXE32ymd6psCaRbwBfhTJnkjCZu_1qgc,3735
3
+ judgeval/constants.py,sha256=JZZJ1MqzZZDVk-5PRPRbmLnM8mXI-RDL5vxa1JFuscs,3408
4
+ judgeval/env.py,sha256=ys0tPuR-ofYC7bm716lCF6zuCRPFU8-W3Tn4QN_DueI,1961
5
+ judgeval/exceptions.py,sha256=WMD_7E9U3SN5uakWYxbrPiAmqn7WKw17dQBTg59LL64,669
6
+ judgeval/judgment_attribute_keys.py,sha256=ko6zM4_2aErXu-LN6NExrrW5HUZjecCNSVsyrPee2cs,2456
7
+ judgeval/logger.py,sha256=8XjpF-92_vDL6Is3YZRBdEPxkynuRBY9QO_O0Efi6H0,1961
8
+ judgeval/version.py,sha256=GAGpb-jQr_yTED9mQSDGKIa_cM-bS2_CnOCg4eFoQ_I,74
9
+ judgeval/warnings.py,sha256=LbGte14ppiFjrkp-JJYueZ40NWFvMkWRvPXr6r-fUWw,73
10
+ judgeval/api/__init__.py,sha256=4Pg0Vk9HxA9O6-XDqKliRjtCZMuIPtpXXAQPJzRIdoQ,15322
11
+ judgeval/api/api_types.py,sha256=Jyflzz7ilWrMpaWlNmKNuHvS-6FSXel1JKy7j0Zb-_4,10173
12
+ judgeval/data/__init__.py,sha256=1tU0EN0ThIfQ1fad5I3dKxAfTcZ5U8cvTLcQ6qLVLU0,407
13
+ judgeval/data/evaluation_run.py,sha256=O41p99wNAuCAf6lsLNKzkZ6W-kL9LlzCYxVls7IcKkA,4727
14
+ judgeval/data/example.py,sha256=eGJpF-lyUH734Cg90B7WtU9f8iKoS3VFGeV6R-GVCCc,1039
15
+ judgeval/data/judgment_types.py,sha256=eJmVVzorMpYpbft9_RYDKf2IYrHEJl_rIqd71pv_PLE,18998
16
+ judgeval/data/result.py,sha256=XufFGSAkBDfevPUmzSgsR9HEqytISkM0U5HkhJmsjpY,2102
17
+ judgeval/data/scorer_data.py,sha256=HeP15ZgftFTJCF8JmDJCLWXRnZJIaGDJCzl7Hg6gWwE,2006
18
+ judgeval/data/trace.py,sha256=zSiR3o6xt8Z46XA3M9fJBtViF0BsPO6yKp9jxdscOSc,3881
19
+ judgeval/data/scripts/fix_default_factory.py,sha256=lvp2JwYZqz-XpD9LZNa3mANZVP-jJSZoNzolI6JWERM,591
20
+ judgeval/data/scripts/openapi_transform.py,sha256=Sm04JClzyP1ga8KA3gkIdsae8Hlx-XU7-x0gHCQYOhg,3877
21
+ judgeval/dataset/__init__.py,sha256=YhfsHA9pUgqQSWs-_QSpNkgOu8sxBsq6Kn3ccbeA_Qg,11162
22
+ judgeval/evaluation/__init__.py,sha256=lZN4krKrY5wGALmf1Rsa3mICLS4tPEtA522hLwKSfYM,13131
23
+ judgeval/integrations/langgraph/__init__.py,sha256=HwXmtDxaO75Kn4KPErnMb6Ne6FcpRxV_SCYVuwFsve0,332
24
+ judgeval/integrations/openlit/__init__.py,sha256=BOCdyFjBrJ2GLE0lsmSXgY7_fEWczeuFQmH9-aRAKuA,1619
25
+ judgeval/judges/__init__.py,sha256=e7JnTc1TG_SwqydDHTXHIP0EBazQxt-ydMQG7ghSU5A,228
26
+ judgeval/judges/base_judge.py,sha256=_dz0qWsKRxzXxpRY9l6mrxTRYPSF2FE4ZXkrzhZ4gbY,986
27
+ judgeval/judges/litellm_judge.py,sha256=5vEF0IUo7HVWnOF2ww-DMke8Xkarnz32B_qbgKjc0-I,4182
28
+ judgeval/judges/together_judge.py,sha256=GzwlXZJzle8hT-vWKmq39JyIeanJqJfHDOkrksUbzk0,4398
29
+ judgeval/judges/utils.py,sha256=ITbYwvjU3o9-FIAReFvxh24yJrx9LV3l9BnSBgKUpxg,2068
30
+ judgeval/prompt/__init__.py,sha256=Qgrd8u4WniaOjbRAoEFEeMnTmaqIGx5ZGX_U85iqhs0,11010
31
+ judgeval/scorers/__init__.py,sha256=pomKzEy4YNFyygYp8vbS3co8iB5CMstRkQwdUgi1u4g,744
32
+ judgeval/scorers/agent_scorer.py,sha256=-qcNSkY6i7ur2LXkM7H1jTKuuFbDuXbjTq42o3vjeQ8,595
33
+ judgeval/scorers/api_scorer.py,sha256=jPBQUBs_T3Xq33QoIbIXDzUaXinz56qeDfo96dfdX0g,2036
34
+ judgeval/scorers/base_scorer.py,sha256=hsMuqdW8QtW5n9JzruXyaZC7im2K2sSmz1RDkbMisJ4,2702
35
+ judgeval/scorers/example_scorer.py,sha256=o_BGUztJXjnKnuOqIa9T4PXe0wPoWg63FyH518N1LxA,561
36
+ judgeval/scorers/exceptions.py,sha256=ACDHK5-TWiF3NTk-wycaedpbrdobm-CvvC1JA_iP-Mk,179
37
+ judgeval/scorers/score.py,sha256=xquM59SCtNeuAsrBsHFgBQk3CHp4-bms4oFs24xfcU0,7176
38
+ judgeval/scorers/utils.py,sha256=dDxPKVjKa1lsMXNhZ8-aJFG3qk1usAH1JnKeC3vBQbU,304
39
+ judgeval/scorers/judgeval_scorers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
40
+ judgeval/scorers/judgeval_scorers/api_scorers/__init__.py,sha256=wrq7y9I30GZbwDXIrSh81KRO_-j7i-1DjwX5Hc3PScI,728
41
+ judgeval/scorers/judgeval_scorers/api_scorers/answer_correctness.py,sha256=WUeFyWdr1Wc8dh-aQ1nrK-mbd9W0MT4VyzLT5CbJ2-Q,450
42
+ judgeval/scorers/judgeval_scorers/api_scorers/answer_relevancy.py,sha256=ciiFBQQC4UDsk9qou9OiKbAR31s82eRUY1ZTt1gdM-0,407
43
+ judgeval/scorers/judgeval_scorers/api_scorers/faithfulness.py,sha256=ucYOI6ztAjfoYmcgTDzN8u5RrehlVqrkeLEfss9b1fk,441
44
+ judgeval/scorers/judgeval_scorers/api_scorers/instruction_adherence.py,sha256=V3RdrWhnR_vLBrtWw7QbgN9K_A-Och7-v9I2fN4z8gY,506
45
+ judgeval/scorers/judgeval_scorers/api_scorers/prompt_scorer.py,sha256=WhSkPs8tWyQ_cS-y-VTzrKAPlizKp-6zi_DmfgW4AgM,10773
46
+ judgeval/tracer/__init__.py,sha256=pxPTipSuuKFOzT0hrM7USTguKu0V-39lfdAloxtNoAM,39296
47
+ judgeval/tracer/constants.py,sha256=tLR5ClDaNlNg_MAv2XRdk62uQW4KyBnWaNbG_YYblTc,55
48
+ judgeval/tracer/keys.py,sha256=mYBo_X6-rC9xfiI-WpjHlO7rUtcMORtQXCQyO1F3Ycc,2387
49
+ judgeval/tracer/managers.py,sha256=JiUjX_evToxcuogKVcE6qpJkSvYOxAXCU4_z_hWXJOw,5199
50
+ judgeval/tracer/utils.py,sha256=xWha5iwC733wCf2HKbNqzxOPS1ovO1OymWIUFLz-UpQ,537
51
+ judgeval/tracer/exporters/__init__.py,sha256=3WDXC28iY5gYMM5s7ejmy7P-DVDQ_iIuzwovZxUKJXg,1295
52
+ judgeval/tracer/exporters/s3.py,sha256=N9gmw17cnR0VkfAQQkLsNj5BksgNRETThR5qYhWRjP4,4360
53
+ judgeval/tracer/exporters/store.py,sha256=pA_KINcm0amO0WEDYmMFU05SSsMOgJ5ogIRaevSX1sk,1885
54
+ judgeval/tracer/exporters/utils.py,sha256=JRcoSQuEHxMDJbXfyrUIfA2SHBVkZM82h4bTbYGxkNw,1154
55
+ judgeval/tracer/llm/__init__.py,sha256=ENxApieKSktYrIviofXWP9GU0WnhBm0Q9mlGe_m_gMY,139
56
+ judgeval/tracer/llm/config.py,sha256=J8-bTL82bgDqdTJPN-Px3Epvoa9FG7L-X329kitwBTc,2525
57
+ judgeval/tracer/llm/constants.py,sha256=IWa3CMes8wIt_UG7jrGEOztg2sHz54fdOMWIOOr-dz8,172
58
+ judgeval/tracer/llm/providers.py,sha256=VAimkmChOOjhC1cUv-0iG8pa5PhOw1HIOyt3zrIrbcM,628
59
+ judgeval/tracer/llm/llm_anthropic/__init__.py,sha256=HG0gIlTgaRt-Y0u1ERPQ19pUgb4YHkTh7tZQPeyR4oM,80
60
+ judgeval/tracer/llm/llm_anthropic/config.py,sha256=ICfKODPQvZsRxpK4xWQ-YE79pmWJTmY2wryddxpNdpM,153
61
+ judgeval/tracer/llm/llm_anthropic/messages.py,sha256=T7dApxJCsOWEpquYSZICACwTioZG3ZcxHdJjvF04T2E,15474
62
+ judgeval/tracer/llm/llm_anthropic/messages_stream.py,sha256=DKlZZnfK_yv_tEMwF2XxvsjgUjOFI3c5JUMQwERNV7k,12188
63
+ judgeval/tracer/llm/llm_anthropic/wrapper.py,sha256=JILcyC4NvjXZSqlFoZp-VB-JsCYZkQPMFEYaB4AysrA,1849
64
+ judgeval/tracer/llm/llm_google/__init__.py,sha256=otBZETsAfVZjtZaN5N36Ln0kw-I9jVB4tFGrV6novHo,74
65
+ judgeval/tracer/llm/llm_google/config.py,sha256=S3yCAE9oHbXjLVYiz5mGD16yIgXMBBUu5UN4lBjoCNQ,162
66
+ judgeval/tracer/llm/llm_google/generate_content.py,sha256=w1rIh1cTBYnkfBQTL4qHntwsKfBcSrf2VSS2y-BOMRU,4030
67
+ judgeval/tracer/llm/llm_google/wrapper.py,sha256=jqaMXGoM9dlPBbCFadMI5EqFrNHzBt0h9VkNn7KPVLk,901
68
+ judgeval/tracer/llm/llm_openai/__init__.py,sha256=CyzwhY0-zmqWKlEno7JPBcvO7G_hI8dp6-_5_KEzFqg,74
69
+ judgeval/tracer/llm/llm_openai/beta_chat_completions.py,sha256=IXw-Gu-WUxQ-gaBUIe-aAKOn1Pakn_RFl0b1C_1toP8,7326
70
+ judgeval/tracer/llm/llm_openai/chat_completions.py,sha256=U086NgaaLFiyvAYrgJncC-obaaSbG2r_3ehquNlVTDQ,17637
71
+ judgeval/tracer/llm/llm_openai/config.py,sha256=NE0ixKhd4WVeAVjY8jNTncuKYH6R4MQDLPmcCsd3zWY,144
72
+ judgeval/tracer/llm/llm_openai/responses.py,sha256=CCGYz35gn3jJOYE2anyR49OR2XhSDwy3dEsISbzMO8Q,18137
73
+ judgeval/tracer/llm/llm_openai/utils.py,sha256=fpy9war8dyke25qHxGW2Yo028RA4Siq0RBLA4G63yUw,1480
74
+ judgeval/tracer/llm/llm_openai/wrapper.py,sha256=Z5Ndib228yd1pXEQ4xIu7_CJHxpW_t0ofZAC6FLc5eU,2055
75
+ judgeval/tracer/llm/llm_together/__init__.py,sha256=MEnsF77IgFD4h73hNCMpo-9a1PHHdm-OxPlOalXOMac,78
76
+ judgeval/tracer/llm/llm_together/chat_completions.py,sha256=RySsK3tqG0NpJHPlVQ705bXxIfseSQUhvIoS-sz4rOg,14380
77
+ judgeval/tracer/llm/llm_together/config.py,sha256=jCJY0KQcHJZZJk2vq038GKIDUMusqgvRjQ0B6OV5uEc,150
78
+ judgeval/tracer/llm/llm_together/wrapper.py,sha256=HFqy_MabQeSq8oj2diZhEuk1SDt_hDfk5MFdPn9MFhg,1733
79
+ judgeval/tracer/processors/__init__.py,sha256=BdOOPOD1RfMI5YHW76DNPKR07EAev-JxoolZ3KaXNNU,7100
80
+ judgeval/trainer/__init__.py,sha256=nJo913vFdss3E_PR-M1OUjznS0SYgNZ-MP-Y_6Mj5PA,437
81
+ judgeval/trainer/base_trainer.py,sha256=Lxm6OxJpifonLKofNIRG3TU7n_jZWQZ0I_f_jwtb_WU,4018
82
+ judgeval/trainer/config.py,sha256=jL8VsjMTPNeRPfhsrOaOiRLjx9BYBc_XnyDw8ncvRqk,4073
83
+ judgeval/trainer/console.py,sha256=SvokkFEU-K1vLV4Rd1m6YJJ7HyYwTr4Azdzwx_JPZUY,4351
84
+ judgeval/trainer/fireworks_trainer.py,sha256=5BvjOWeRA6DOWez861ewWgRYyqpnMG22Sxj2Hqz4L50,15933
85
+ judgeval/trainer/trainable_model.py,sha256=96XI2_y1M8LiT7xZJ9Ydu7BiRn_iAGj6MI5n_orD2iA,9433
86
+ judgeval/trainer/trainer.py,sha256=twLEHNaomelTg6ZYG6veI9OpB3wzhPCtPVQMTnDZWx4,2626
87
+ judgeval/utils/async_utils.py,sha256=AF1xdu8Ao5GyhFvfaLOaKJHn1RISyXZ4U70UZe9zfBA,1083
88
+ judgeval/utils/file_utils.py,sha256=vq-n5WZEZjVbZ5S9QTkW8nSH6Pvw-Jx0ttsQ1t0wnPQ,3140
89
+ judgeval/utils/guards.py,sha256=_DaKZxvjD10J97Ze2paHhbCiV2MpDz3FZQmNwaL5k0w,945
90
+ judgeval/utils/meta.py,sha256=RAqZuvOlymqMwFoS0joBW_r65lcN9bY8BpNYHoytKps,773
91
+ judgeval/utils/project.py,sha256=kGpYmp6QGTD6h-GjQ-ovT7kBmGnyb99MWDJmRGFQHOg,527
92
+ judgeval/utils/serialize.py,sha256=WbforbVFGINuk68T2YtWhj-ECMC6rWol3g5dxz9nsm8,6265
93
+ judgeval/utils/testing.py,sha256=m5Nexv65tmfSj1XvAPK5Ear7aJ7w5xjDtZN0tLZ_RBk,2939
94
+ judgeval/utils/url.py,sha256=Shf0v3XcbaWpL0m1eGJEEO_z4TsQCnDB2Rl25OTUmiI,195
95
+ judgeval/utils/version_check.py,sha256=se4Ft8rjcl5u7fHMxSGQpka844V2AcZpOYl6StLWTio,1081
96
+ judgeval/utils/decorators/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
97
+ judgeval/utils/decorators/dont_throw.py,sha256=B6N-4RhHxMn55TqMImVGG82BVzh3yOPmcrmSflT7f4I,989
98
+ judgeval/utils/decorators/use_once.py,sha256=8mgj5VK9v08VOOWX2Bstc0CezNsOVUKMIv7N2R83E8s,288
99
+ judgeval/utils/wrappers/README.md,sha256=-Jyagu6NPH92ty8pTMbzRLVJZzufULrjxcyohXgsGMc,76
100
+ judgeval/utils/wrappers/__init__.py,sha256=iAcpjCOkYqoe6z2utrS_3yZLmdQPD1Y64MMefai8h0Y,546
101
+ judgeval/utils/wrappers/immutable_wrap_async.py,sha256=a0LWyEa235tPpfjN3W0A516_GWDL13uhumMVzsMpgW4,1909
102
+ judgeval/utils/wrappers/immutable_wrap_async_iterator.py,sha256=hfVxBoFE6m6I0g0KcSLJXyfVv6pfZuoJuHyLJHtLjjg,2268
103
+ judgeval/utils/wrappers/immutable_wrap_sync.py,sha256=_gOUaPK4Le-pifWCZOH4lDvY-cLfYoC0fy7DTNIG-0A,1823
104
+ judgeval/utils/wrappers/immutable_wrap_sync_iterator.py,sha256=aDC4HpLp4l9A3aFLS0cTCkien-xGgQRU04F7P1pJ6w8,2229
105
+ judgeval/utils/wrappers/mutable_wrap_async.py,sha256=stHISOUCGFUJXY8seXmxUo4ZpMF4LErSBIz0HlWR7Bo,2941
106
+ judgeval/utils/wrappers/mutable_wrap_sync.py,sha256=t5jygAQ1vqhy8s1GfiLeYygYgaLTgfoYASN47U5JiPs,2888
107
+ judgeval/utils/wrappers/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
108
+ judgeval/utils/wrappers/utils.py,sha256=j18vaa6JWDw2s3nQy1z5PfV_9Xxio-bVARaHG_0XyL0,1228
109
+ judgeval/v1/__init__.py,sha256=xfEruFmFiMC0gqp3IcJynOMqo5h8nqiZLgCLZpjsEJU,2322
110
+ judgeval/v1/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
111
+ judgeval/v1/utils.py,sha256=Vw34Ouzmos6yuTyrV2jHY9Ym9jbBApOOZ3--7bYPt7g,632
112
+ judgeval/v1/data/__init__.py,sha256=hPTyLqn5XViwOnYmNL17YUkyVmSxx42mv4JDxmrQcTk,245
113
+ judgeval/v1/data/example.py,sha256=Q4N-GHQeEAIfVd4_GMaV6MCZHCzRYZOPPtsBvBh3RW0,1355
114
+ judgeval/v1/data/scorer_data.py,sha256=mYip_rz7HfG12sd_Z61ZKUGzZpjpZHgbJBRh5h3eUlA,1391
115
+ judgeval/v1/data/scoring_result.py,sha256=u4PJzCcCEl-I8L0GZtkIp_LZIV0mZrliWxWximSHz6Q,1540
116
+ judgeval/v1/datasets/__init__.py,sha256=Q0R1n993aKLp6sy5wmYDYfrU4wsWjBsBuPmx3EGXncQ,218
117
+ judgeval/v1/datasets/dataset.py,sha256=LP3dBYanJLWODdVVH0eGF9CKPR2hkoLXsUw0SigjeJ8,6935
118
+ judgeval/v1/datasets/dataset_factory.py,sha256=bI6ibn37gdQicuLW1lRXGsq5xsCcCDsYVKK6j8Zk_yQ,3202
119
+ judgeval/v1/evaluation/__init__.py,sha256=5zN14e_Ukmx53Q-BnR1KzV3GbhBrS372R62SAJudR9U,212
120
+ judgeval/v1/evaluation/evaluation.py,sha256=k2Jk6o5LYrUy8ye6UktXJBUemBGHQ440MU7xp93RQUI,6738
121
+ judgeval/v1/evaluation/evaluation_factory.py,sha256=kp-YSAVjdpTFMouCg3aMpDakHpLmvYRHduBphfCDJ1Y,392
122
+ judgeval/v1/instrumentation/__init__.py,sha256=7kmEJrqVsKsFqOAXNywRLzQZ6irmtvEGZhb7zLBa0E0,85
123
+ judgeval/v1/instrumentation/llm/__init__.py,sha256=ENxApieKSktYrIviofXWP9GU0WnhBm0Q9mlGe_m_gMY,139
124
+ judgeval/v1/instrumentation/llm/config.py,sha256=CyxJ90GuxcqVUqkucuA6F2fWhgfVNJX1tIO2-DXirx0,2572
125
+ judgeval/v1/instrumentation/llm/constants.py,sha256=1BaM6XEzrCu-DCUeUXxu-p0K5XyWqBXeFQ-Ks_9mp0g,208
126
+ judgeval/v1/instrumentation/llm/providers.py,sha256=KdPNi0G69aG1OfIlXy7PoGLx10Z8n0jqVLKNFZi7RQI,676
127
+ judgeval/v1/instrumentation/llm/llm_anthropic/__init__.py,sha256=iqv1FpU6pldnPRLlhaq5_H1CQbNnquzB3mcWqpTJ68w,116
128
+ judgeval/v1/instrumentation/llm/llm_anthropic/config.py,sha256=ICfKODPQvZsRxpK4xWQ-YE79pmWJTmY2wryddxpNdpM,153
129
+ judgeval/v1/instrumentation/llm/llm_anthropic/messages.py,sha256=8JAwMnYxeUlYLP2zpPIIPmQ2e6zveYBundrDdxKJ-Mc,14823
130
+ judgeval/v1/instrumentation/llm/llm_anthropic/messages_stream.py,sha256=JiB3e1GkAnITuzW6pi7i0xyiBk_TCSqQuDkrIvpLNGA,11766
131
+ judgeval/v1/instrumentation/llm/llm_anthropic/wrapper.py,sha256=vW5UeS0uabF-Rn7hVrpPSiUcM2hYV4naZYcOXgnO8D4,1918
132
+ judgeval/v1/instrumentation/llm/llm_google/__init__.py,sha256=-LUKGeKGFXTveWLSODbM_42hdFbyLEWbapfh6LT-j1s,110
133
+ judgeval/v1/instrumentation/llm/llm_google/config.py,sha256=S3yCAE9oHbXjLVYiz5mGD16yIgXMBBUu5UN4lBjoCNQ,162
134
+ judgeval/v1/instrumentation/llm/llm_google/generate_content.py,sha256=UP6rCCJoa68nZBD0NqbJBlscSJuRSBhK39qs1FxzQD0,3916
135
+ judgeval/v1/instrumentation/llm/llm_google/wrapper.py,sha256=z7EECZ8c3nEKFTkXaCCBFPog3OeH__NaiHQulGBpS1g,936
136
+ judgeval/v1/instrumentation/llm/llm_openai/__init__.py,sha256=fUnFiufoikJkA6hkLV0UUljBIt42lR5Xy0wx5sIXfMg,110
137
+ judgeval/v1/instrumentation/llm/llm_openai/beta_chat_completions.py,sha256=PO0mdOFjKsLe8wa4d9sJ4JNVkK9fuw9CswX0scYb0VE,7229
138
+ judgeval/v1/instrumentation/llm/llm_openai/chat_completions.py,sha256=G3z-O7FvmhsOg3RcjfTUwQCmO_H1F9LecfYJuDmTD7g,17224
139
+ judgeval/v1/instrumentation/llm/llm_openai/config.py,sha256=NE0ixKhd4WVeAVjY8jNTncuKYH6R4MQDLPmcCsd3zWY,144
140
+ judgeval/v1/instrumentation/llm/llm_openai/responses.py,sha256=FS81UOe63ea0-eKEfh2nY958nvLWyDDG3RhTsNe2hqA,17382
141
+ judgeval/v1/instrumentation/llm/llm_openai/utils.py,sha256=l_81Om6IYel7fxYSwGYxFwHncZhjKM4oucXIAGg6kFE,1357
142
+ judgeval/v1/instrumentation/llm/llm_openai/wrapper.py,sha256=cY8Y3H4mLm6OT1UleHVArKFpuQdBf0iI_JJJLOz07Yo,2130
143
+ judgeval/v1/instrumentation/llm/llm_together/__init__.py,sha256=HsZB7HdnbWANuLyH-eUXze-gzoV2oJaBzEMzvcRilu4,114
144
+ judgeval/v1/instrumentation/llm/llm_together/chat_completions.py,sha256=oFKBb8wc_NJyYFxg-XRnEo5ZrpqK_FWhjNYH8WpgO0M,14021
145
+ judgeval/v1/instrumentation/llm/llm_together/config.py,sha256=jCJY0KQcHJZZJk2vq038GKIDUMusqgvRjQ0B6OV5uEc,150
146
+ judgeval/v1/instrumentation/llm/llm_together/wrapper.py,sha256=fSB3gt9S-NWnzaTJUiRtiblbojg6sRftGlIeENv8RK8,1801
147
+ judgeval/v1/integrations/claude_agent_sdk/__init__.py,sha256=5ozUauaDwqqzHGpJifcAEwTLeL83_YEYJtkiU0JGg3E,4552
148
+ judgeval/v1/integrations/claude_agent_sdk/wrapper.py,sha256=san7IhyTHmX0LpEQhsHiQWeidcJDKUJ2LxmnRm0Vg7Q,21496
149
+ judgeval/v1/integrations/langgraph/__init__.py,sha256=HwXmtDxaO75Kn4KPErnMb6Ne6FcpRxV_SCYVuwFsve0,332
150
+ judgeval/v1/integrations/openlit/__init__.py,sha256=Eq-RRKQiDkzNslPjXy_EdpkHTF5Kwd1XnGluVMxOZuk,1400
151
+ judgeval/v1/internal/api/__init__.py,sha256=8NzoAyfYfeCj04F9nB5hCYmi4ar-A6dD9T4ZPUwWxrU,16174
152
+ judgeval/v1/internal/api/api_types.py,sha256=EIbnT8NqEh5Rt6kF6r0hu394vUdBga2lq1Awl47GXJE,10176
153
+ judgeval/v1/prompts/__init__.py,sha256=_en7nUIQzhicHAmLMNT9bcRVhdUiaIizWofhok-V8iI,182
154
+ judgeval/v1/prompts/prompt.py,sha256=AW5XFPQIDEAFHH2uqbkUUN5zvpgjFJJ7MPFAJIXhr4M,835
155
+ judgeval/v1/prompts/prompt_factory.py,sha256=S5PmJYyep_zRNvcwSaxCFn7e1HHT7_rYEIZwJSQc-jY,5689
156
+ judgeval/v1/scorers/__init__.py,sha256=GIzpQmcbsDsIvJs4CO2baata1QzwYAdk1N-GdywJ_9Y,198
157
+ judgeval/v1/scorers/api_scorer.py,sha256=m6KXtrEejKlY125EwYSsun31fIMABiYXAOqwuOqYfZA,2360
158
+ judgeval/v1/scorers/base_scorer.py,sha256=nhem2oU4_OkS5-IHVVepe1lsNalu_VzdBWyIP3SP5XE,325
159
+ judgeval/v1/scorers/scorers_factory.py,sha256=Hk2iyx5yhdki4fVwBHyNxouJZAoaazTyh9M-02qrSgA,1169
160
+ judgeval/v1/scorers/built_in/__init__.py,sha256=H6CS3q_k5G42lQgswCDYbyolTelE1cvgTsPFYJPGmsY,616
161
+ judgeval/v1/scorers/built_in/answer_correctness.py,sha256=PqRsLOFAPy84N0o8ow59bH0CrL-fabSZ9dSWothF3yA,824
162
+ judgeval/v1/scorers/built_in/answer_relevancy.py,sha256=oQGyNSkUNSlcIPa41a6vy31ZKxcE3ONE29hp6IlvufQ,797
163
+ judgeval/v1/scorers/built_in/built_in_factory.py,sha256=ptdSEya2-TJWArMDJvXU-ECsE1Xmd7uXvdJfO17Ra8A,995
164
+ judgeval/v1/scorers/built_in/faithfulness.py,sha256=YwT4J4B8e6vW5gfJfpSzUaet20xpzyKiCVyG36PfSMY,786
165
+ judgeval/v1/scorers/built_in/instruction_adherence.py,sha256=5vOi6zXewryK5L_1hHNnhQbVRTqusH5ftCcoXs36k0M,817
166
+ judgeval/v1/scorers/custom_scorer/__init__.py,sha256=LqIDe6Lftv1DiX-sY6maN7fKUjFiXWNj6Gdk66O3gyk,248
167
+ judgeval/v1/scorers/custom_scorer/custom_scorer.py,sha256=mLQ-ZKzjGX78aky2F44ipQCBjQ6Fw2hlkUxabZ8wirU,1257
168
+ judgeval/v1/scorers/custom_scorer/custom_scorer_factory.py,sha256=BBCGDuxRPff1Xv_nJFqJrxss__Dk5p1jOhd5aYOqBiI,404
169
+ judgeval/v1/scorers/prompt_scorer/__init__.py,sha256=lXu2aIRKzgPqBzai9paMdx8sGiUu9k6-GeTS15OJNGk,248
170
+ judgeval/v1/scorers/prompt_scorer/prompt_scorer.py,sha256=spl3ISlPkp9DP6feavuWjK18yAa0SjxWPfzO9KfDpA8,2494
171
+ judgeval/v1/scorers/prompt_scorer/prompt_scorer_factory.py,sha256=Rg7hPltO1zk0SbkkgA4d5vOhk1S_FKm_r7nrKZ64HVU,2821
172
+ judgeval/v1/tracer/__init__.py,sha256=EkmERuaDI3PTjwy9L5HB7UCfuEJ-tqSzDGTR_NG5Y1M,248
173
+ judgeval/v1/tracer/base_tracer.py,sha256=vp97UTm-dSi9-HpBw2OAVq53G9bN2uHGdB2DRTBPCGQ,18327
174
+ judgeval/v1/tracer/judgment_tracer_provider.py,sha256=i4lKcbc209-6XN-ymUR0ugRLTEw8R0_l6bwbDybmzvs,2358
175
+ judgeval/v1/tracer/tracer.py,sha256=H71bvnn39YtA5VuQAYydst6y_1FncZuhdCyEd1HW9fA,2327
176
+ judgeval/v1/tracer/tracer_factory.py,sha256=0MRtbvoLpb8aQuDqOeobxu-dTVKrLgQBCHUFjZqMVwQ,1075
177
+ judgeval/v1/tracer/exporters/__init__.py,sha256=5dafftPbJlzBQQUSat44I8oZF0XLo1D_Ulsf2A9D3QY,492
178
+ judgeval/v1/tracer/exporters/in_memory_span_exporter.py,sha256=qc7PCnxdxt0Sx2GyysOHe5AMmG2p3sDhEoalAVjvSaI,685
179
+ judgeval/v1/tracer/exporters/judgment_span_exporter.py,sha256=C9P4tZR5Qtlhrv-Xbjpi05way9Irx54wJnniAFXHJ9Q,1313
180
+ judgeval/v1/tracer/exporters/noop_span_exporter.py,sha256=q_xwtfnULw34ENYDWOiN4OsbOq_l0teqh2GCWB0oiUI,492
181
+ judgeval/v1/tracer/exporters/span_store.py,sha256=YmzDX_uCnRn_Y6J-oXSX0hDAvhI1lSCW5PJjBjBBVQE,1503
182
+ judgeval/v1/tracer/processors/__init__.py,sha256=5XqsiixgxFG7X4AzmFLYmiGYCV7FtrW5JWzjyPAEPzg,278
183
+ judgeval/v1/tracer/processors/judgment_span_processor.py,sha256=-pYbJRV_wBEJ_zmFlx7gzp0wSuzqieaDWh3SYDlq0zk,5421
184
+ judgeval/v1/tracer/processors/noop_span_processor.py,sha256=zx5ZcfRFqWkPh_ClLO5qXnYLxUW6aE1NEVSHx65-5mk,1129
185
+ judgeval/v1/tracer/processors/_lifecycles/__init__.py,sha256=tiZ2__ccbRsm4eJCNPbtL-_KYZjJY8pwkkGyqdUmQOA,737
186
+ judgeval/v1/tracer/processors/_lifecycles/agent_id_processor.py,sha256=KW0tmMpVXiOCvOxqsKjbR3HRA7DsW7djHDL38Jj1W70,1861
187
+ judgeval/v1/tracer/processors/_lifecycles/context_keys.py,sha256=yRV9GfPELOr6b2KlVOW4VhVnJJ0oxzW6WnV5Q0gLc7s,495
188
+ judgeval/v1/tracer/processors/_lifecycles/customer_id_processor.py,sha256=afWFAYvfJG89EdD4K_ENaA1sXU0urzkd-tQp-9fh2fk,954
189
+ judgeval/v1/tracer/processors/_lifecycles/registry.py,sha256=24AiMjKts65TlE2vF6xYNtgmsycZ9gpnoylHVOzIdsw,431
190
+ judgeval/v1/trainers/__init__.py,sha256=VUmT_y-zwWmAmtv_IjUNiNwr4BjdZK1-LNor7OkcSE0,133
191
+ judgeval/v1/trainers/base_trainer.py,sha256=sMe1_lFIPhLtr9ck5WwbYiiPD-2w2fHbI-hJ1KOnt-Y,1835
192
+ judgeval/v1/trainers/config.py,sha256=jL8VsjMTPNeRPfhsrOaOiRLjx9BYBc_XnyDw8ncvRqk,4073
193
+ judgeval/v1/trainers/console.py,sha256=SvokkFEU-K1vLV4Rd1m6YJJ7HyYwTr4Azdzwx_JPZUY,4351
194
+ judgeval/v1/trainers/fireworks_trainer.py,sha256=Jx41Kbgfwl8-pR8u3WGZvrrmviFWwzfvgwlVxoD5Sn0,16110
195
+ judgeval/v1/trainers/trainable_model.py,sha256=96XI2_y1M8LiT7xZJ9Ydu7BiRn_iAGj6MI5n_orD2iA,9433
196
+ judgeval/v1/trainers/trainers_factory.py,sha256=a6eABP_FZiP0x3jMBNOKlM-EKcDiQvQS4d6YZ0PedQU,946
197
+ judgeval-0.23.0.dist-info/METADATA,sha256=Len3xf4OuZOkR0CmMbo4KLfDVTxmV7kkwecHB3ucfBw,11598
198
+ judgeval-0.23.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
199
+ judgeval-0.23.0.dist-info/entry_points.txt,sha256=-eoeD-oDLn4A7MSgeBS9Akwanf3_0r0cgEleBcIOjg0,46
200
+ judgeval-0.23.0.dist-info/licenses/LICENSE.md,sha256=tKmCg7k5QOmxPK19XMfzim04QiQJPmgIm0pAn55IJwk,11352
201
+ judgeval-0.23.0.dist-info/RECORD,,
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ judgeval = judgeval.cli:app
judgeval/clients.py DELETED
@@ -1,34 +0,0 @@
1
- import os
2
- from dotenv import load_dotenv
3
- from openai import OpenAI
4
- from typing import Optional
5
- from together import Together, AsyncTogether
6
-
7
- PATH_TO_DOTENV = os.path.join(os.path.dirname(__file__), ".env")
8
- load_dotenv(dotenv_path=PATH_TO_DOTENV)
9
-
10
-
11
- # Initialize optional OpenAI client
12
- client: Optional["OpenAI"] = None
13
- if os.getenv("OPENAI_API_KEY"):
14
- try:
15
- from openai import OpenAI
16
-
17
- client = OpenAI()
18
- except ImportError:
19
- # openai package not installed
20
- pass
21
-
22
- # Initialize optional Together clients
23
- together_client: Optional["Together"] = None
24
- async_together_client: Optional["AsyncTogether"] = None
25
-
26
- # Only initialize Together clients if API key is available
27
-
28
- together_api_key = os.getenv("TOGETHERAI_API_KEY") or os.getenv("TOGETHER_API_KEY")
29
- if together_api_key:
30
- try:
31
- together_client = Together(api_key=together_api_key)
32
- async_together_client = AsyncTogether(api_key=together_api_key)
33
- except Exception:
34
- pass
@@ -1,13 +0,0 @@
1
- from judgeval.common.utils import (
2
- get_chat_completion,
3
- aget_chat_completion,
4
- get_completion_multiple_models,
5
- aget_completion_multiple_models,
6
- )
7
-
8
- __all__ = [
9
- "get_chat_completion",
10
- "aget_chat_completion",
11
- "get_completion_multiple_models",
12
- "aget_completion_multiple_models",
13
- ]
@@ -1,3 +0,0 @@
1
- from .api import JudgmentApiClient, JudgmentAPIException
2
-
3
- __all__ = ["JudgmentApiClient", "JudgmentAPIException"]
@@ -1,352 +0,0 @@
1
- from typing import Literal, List, Dict, Any
2
- from requests import exceptions
3
- from judgeval.common.api.constants import (
4
- JUDGMENT_TRACES_FETCH_API_URL,
5
- JUDGMENT_TRACES_UPSERT_API_URL,
6
- JUDGMENT_TRACES_DELETE_API_URL,
7
- JUDGMENT_TRACES_SPANS_BATCH_API_URL,
8
- JUDGMENT_TRACES_EVALUATION_RUNS_BATCH_API_URL,
9
- JUDGMENT_DATASETS_PUSH_API_URL,
10
- JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL,
11
- JUDGMENT_DATASETS_PULL_API_URL,
12
- JUDGMENT_DATASETS_DELETE_API_URL,
13
- JUDGMENT_DATASETS_PROJECT_STATS_API_URL,
14
- JUDGMENT_PROJECT_DELETE_API_URL,
15
- JUDGMENT_PROJECT_CREATE_API_URL,
16
- JUDGMENT_EVAL_API_URL,
17
- JUDGMENT_TRACE_EVAL_API_URL,
18
- JUDGMENT_EVAL_LOG_API_URL,
19
- JUDGMENT_EVAL_FETCH_API_URL,
20
- JUDGMENT_EVAL_DELETE_API_URL,
21
- JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL,
22
- JUDGMENT_GET_EVAL_STATUS_API_URL,
23
- JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL,
24
- JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL,
25
- JUDGMENT_SCORER_SAVE_API_URL,
26
- JUDGMENT_SCORER_FETCH_API_URL,
27
- JUDGMENT_SCORER_EXISTS_API_URL,
28
- )
29
- from judgeval.common.api.constants import (
30
- TraceFetchPayload,
31
- TraceDeletePayload,
32
- SpansBatchPayload,
33
- EvaluationEntryResponse,
34
- EvaluationRunsBatchPayload,
35
- DatasetPushPayload,
36
- DatasetAppendPayload,
37
- DatasetPullPayload,
38
- DatasetDeletePayload,
39
- DatasetStatsPayload,
40
- ProjectCreatePayload,
41
- ProjectDeletePayload,
42
- EvalRunRequestBody,
43
- DeleteEvalRunRequestBody,
44
- EvalLogPayload,
45
- EvalStatusPayload,
46
- CheckExperimentTypePayload,
47
- EvalRunNameExistsPayload,
48
- ScorerSavePayload,
49
- ScorerFetchPayload,
50
- ScorerExistsPayload,
51
- )
52
- from judgeval.utils.requests import requests
53
-
54
-
55
- class JudgmentAPIException(exceptions.HTTPError):
56
- """
57
- Exception raised when an error occurs while executing a Judgment API request.
58
- Extends requests.exceptions.HTTPError to provide access to the response object.
59
- """
60
-
61
- def __init__(self, message: str, response=None, request=None):
62
- super().__init__(message, response=response, request=request)
63
- self.message = message
64
- self.response = response
65
- self.request = request
66
-
67
- @property
68
- def status_code(self) -> int | None:
69
- """Get the HTTP status code from the response."""
70
- return self.response.status_code if self.response else None
71
-
72
- @property
73
- def response_json(self) -> Dict[str, Any]:
74
- """Get the JSON response body."""
75
- try:
76
- return self.response.json() if self.response else {}
77
- except (ValueError, AttributeError):
78
- return {}
79
-
80
- @property
81
- def error_detail(self) -> str:
82
- """Get the error detail from the response JSON."""
83
- return self.response_json.get("detail", "An unknown error occurred.")
84
-
85
-
86
- class JudgmentApiClient:
87
- def __init__(self, api_key: str, organization_id: str):
88
- self.api_key = api_key
89
- self.organization_id = organization_id
90
-
91
- def _do_request(
92
- self,
93
- method: Literal["POST", "PATCH", "GET", "DELETE"],
94
- url: str,
95
- payload: Any,
96
- ) -> Any:
97
- if method == "GET":
98
- r = requests.request(
99
- method,
100
- url,
101
- params=payload,
102
- headers=self._headers(),
103
- **self._request_kwargs(),
104
- )
105
- else:
106
- r = requests.request(
107
- method,
108
- url,
109
- data=self._serialize(payload),
110
- headers=self._headers(),
111
- **self._request_kwargs(),
112
- )
113
-
114
- try:
115
- r.raise_for_status()
116
- except exceptions.HTTPError as e:
117
- raise JudgmentAPIException(
118
- f"HTTP {r.status_code}: {r.reason}", response=r, request=e.request
119
- )
120
-
121
- return r.json()
122
-
123
- def send_spans_batch(self, spans: List[Dict[str, Any]]):
124
- payload: SpansBatchPayload = {
125
- "spans": spans,
126
- "organization_id": self.organization_id,
127
- }
128
-
129
- return self._do_request("POST", JUDGMENT_TRACES_SPANS_BATCH_API_URL, payload)
130
-
131
- def send_evaluation_runs_batch(
132
- self, evaluation_entries: List[EvaluationEntryResponse]
133
- ):
134
- payload: EvaluationRunsBatchPayload = {
135
- "organization_id": self.organization_id,
136
- "evaluation_entries": evaluation_entries,
137
- }
138
-
139
- return self._do_request(
140
- "POST", JUDGMENT_TRACES_EVALUATION_RUNS_BATCH_API_URL, payload
141
- )
142
-
143
- def fetch_trace(self, trace_id: str):
144
- payload: TraceFetchPayload = {"trace_id": trace_id}
145
- return self._do_request("POST", JUDGMENT_TRACES_FETCH_API_URL, payload)
146
-
147
- def upsert_trace(self, trace_data: Dict[str, Any]):
148
- return self._do_request("POST", JUDGMENT_TRACES_UPSERT_API_URL, trace_data)
149
-
150
- def delete_trace(self, trace_id: str):
151
- payload: TraceDeletePayload = {"trace_ids": [trace_id]}
152
- return self._do_request("DELETE", JUDGMENT_TRACES_DELETE_API_URL, payload)
153
-
154
- def delete_traces(self, trace_ids: List[str]):
155
- payload: TraceDeletePayload = {"trace_ids": trace_ids}
156
- return self._do_request("DELETE", JUDGMENT_TRACES_DELETE_API_URL, payload)
157
-
158
- def delete_project(self, project_name: str):
159
- payload: ProjectDeletePayload = {"project_name": project_name}
160
- return self._do_request("DELETE", JUDGMENT_PROJECT_DELETE_API_URL, payload)
161
-
162
- def create_project(self, project_name: str):
163
- payload: ProjectCreatePayload = {"project_name": project_name}
164
- return self._do_request("POST", JUDGMENT_PROJECT_CREATE_API_URL, payload)
165
-
166
- def run_evaluation(self, evaluation_run: Dict[str, Any]):
167
- return self._do_request("POST", JUDGMENT_EVAL_API_URL, evaluation_run)
168
-
169
- def run_trace_evaluation(self, trace_run: Dict[str, Any]):
170
- return self._do_request("POST", JUDGMENT_TRACE_EVAL_API_URL, trace_run)
171
-
172
- def log_evaluation_results(
173
- self, results: List[Dict[str, Any]], run: Dict[str, Any]
174
- ):
175
- payload: EvalLogPayload = {"results": results, "run": run}
176
- return self._do_request("POST", JUDGMENT_EVAL_LOG_API_URL, payload)
177
-
178
- def fetch_evaluation_results(self, project_name: str, eval_name: str):
179
- payload: EvalRunRequestBody = {
180
- "project_name": project_name,
181
- "eval_name": eval_name,
182
- }
183
- return self._do_request("POST", JUDGMENT_EVAL_FETCH_API_URL, payload)
184
-
185
- def delete_evaluation_results(self, project_name: str, eval_names: List[str]):
186
- payload: DeleteEvalRunRequestBody = {
187
- "project_name": project_name,
188
- "eval_names": eval_names,
189
- "judgment_api_key": self.api_key,
190
- }
191
- return self._do_request("POST", JUDGMENT_EVAL_DELETE_API_URL, payload)
192
-
193
- def add_to_evaluation_queue(self, payload: Dict[str, Any]):
194
- return self._do_request("POST", JUDGMENT_ADD_TO_RUN_EVAL_QUEUE_API_URL, payload)
195
-
196
- def get_evaluation_status(self, eval_name: str, project_name: str):
197
- payload: EvalStatusPayload = {
198
- "eval_name": eval_name,
199
- "project_name": project_name,
200
- "judgment_api_key": self.api_key,
201
- }
202
- return self._do_request("GET", JUDGMENT_GET_EVAL_STATUS_API_URL, payload)
203
-
204
- def check_experiment_type(self, eval_name: str, project_name: str, is_trace: bool):
205
- payload: CheckExperimentTypePayload = {
206
- "eval_name": eval_name,
207
- "project_name": project_name,
208
- "judgment_api_key": self.api_key,
209
- "is_trace": is_trace,
210
- }
211
- return self._do_request("POST", JUDGMENT_CHECK_EXPERIMENT_TYPE_API_URL, payload)
212
-
213
- def check_eval_run_name_exists(self, eval_name: str, project_name: str):
214
- payload: EvalRunNameExistsPayload = {
215
- "eval_name": eval_name,
216
- "project_name": project_name,
217
- "judgment_api_key": self.api_key,
218
- }
219
- return self._do_request("POST", JUDGMENT_EVAL_RUN_NAME_EXISTS_API_URL, payload)
220
-
221
- def save_scorer(self, name: str, prompt: str, options: dict):
222
- payload: ScorerSavePayload = {
223
- "name": name,
224
- "prompt": prompt,
225
- "options": options,
226
- }
227
- try:
228
- return self._do_request("POST", JUDGMENT_SCORER_SAVE_API_URL, payload)
229
- except JudgmentAPIException as e:
230
- if e.status_code == 500:
231
- raise JudgmentAPIException(
232
- f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}",
233
- response=e.response,
234
- request=e.request,
235
- )
236
- raise JudgmentAPIException(
237
- f"Failed to save classifier scorer: {e.error_detail}",
238
- response=e.response,
239
- request=e.request,
240
- )
241
-
242
- def fetch_scorer(self, name: str):
243
- payload: ScorerFetchPayload = {"name": name}
244
- try:
245
- return self._do_request("POST", JUDGMENT_SCORER_FETCH_API_URL, payload)
246
- except JudgmentAPIException as e:
247
- if e.status_code == 500:
248
- raise JudgmentAPIException(
249
- f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}",
250
- response=e.response,
251
- request=e.request,
252
- )
253
- raise JudgmentAPIException(
254
- f"Failed to fetch classifier scorer '{name}': {e.error_detail}",
255
- response=e.response,
256
- request=e.request,
257
- )
258
-
259
- def scorer_exists(self, name: str):
260
- payload: ScorerExistsPayload = {"name": name}
261
- try:
262
- return self._do_request("POST", JUDGMENT_SCORER_EXISTS_API_URL, payload)
263
- except JudgmentAPIException as e:
264
- if e.status_code == 500:
265
- raise JudgmentAPIException(
266
- f"The server is temporarily unavailable. Please try your request again in a few moments. Error details: {e.error_detail}",
267
- response=e.response,
268
- request=e.request,
269
- )
270
- raise JudgmentAPIException(
271
- f"Failed to check if scorer exists: {e.error_detail}",
272
- response=e.response,
273
- request=e.request,
274
- )
275
-
276
- def push_dataset(
277
- self,
278
- dataset_alias: str,
279
- project_name: str,
280
- examples: List[Dict[str, Any]],
281
- traces: List[Dict[str, Any]],
282
- overwrite: bool,
283
- ):
284
- payload: DatasetPushPayload = {
285
- "dataset_alias": dataset_alias,
286
- "project_name": project_name,
287
- "examples": examples,
288
- "traces": traces,
289
- "overwrite": overwrite,
290
- }
291
- return self._do_request("POST", JUDGMENT_DATASETS_PUSH_API_URL, payload)
292
-
293
- def append_examples(
294
- self, dataset_alias: str, project_name: str, examples: List[Dict[str, Any]]
295
- ):
296
- payload: DatasetAppendPayload = {
297
- "dataset_alias": dataset_alias,
298
- "project_name": project_name,
299
- "examples": examples,
300
- }
301
- return self._do_request(
302
- "POST", JUDGMENT_DATASETS_APPEND_EXAMPLES_API_URL, payload
303
- )
304
-
305
- def pull_dataset(self, dataset_alias: str, project_name: str):
306
- payload: DatasetPullPayload = {
307
- "dataset_alias": dataset_alias,
308
- "project_name": project_name,
309
- }
310
- return self._do_request("POST", JUDGMENT_DATASETS_PULL_API_URL, payload)
311
-
312
- def delete_dataset(self, dataset_alias: str, project_name: str):
313
- payload: DatasetDeletePayload = {
314
- "dataset_alias": dataset_alias,
315
- "project_name": project_name,
316
- }
317
- return self._do_request("POST", JUDGMENT_DATASETS_DELETE_API_URL, payload)
318
-
319
- def get_project_dataset_stats(self, project_name: str):
320
- payload: DatasetStatsPayload = {"project_name": project_name}
321
- return self._do_request(
322
- "POST", JUDGMENT_DATASETS_PROJECT_STATS_API_URL, payload
323
- )
324
-
325
- def _headers(self) -> Dict[str, str]:
326
- return {
327
- "Content-Type": "application/json",
328
- "Authorization": f"Bearer {self.api_key}",
329
- "X-Organization-Id": self.organization_id,
330
- }
331
-
332
- def _request_kwargs(self):
333
- # NOTE: We may want to configure custom kwargs that different requests may need.
334
- # For this purpose we can store that as a property of self, and return the appropriate kwargs from this method.
335
- return {
336
- "verify": True,
337
- "timeout": 30,
338
- }
339
-
340
- def _serialize(self, data: Any) -> str:
341
- def fallback_encoder(obj):
342
- try:
343
- return repr(obj)
344
- except Exception:
345
- try:
346
- return str(obj)
347
- except Exception as e:
348
- return f"<Unserializable object of type {type(obj).__name__}: {e}>"
349
-
350
- import json
351
-
352
- return json.dumps(data, default=fallback_encoder)