deepeval 3.6.4__py3-none-any.whl → 3.6.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- deepeval/__init__.py +42 -10
- deepeval/_version.py +1 -1
- deepeval/config/logging.py +33 -0
- deepeval/config/settings.py +167 -12
- deepeval/dataset/dataset.py +8 -2
- deepeval/evaluate/evaluate.py +8 -2
- deepeval/evaluate/execute.py +28 -30
- deepeval/evaluate/types.py +4 -1
- deepeval/evaluate/utils.py +46 -29
- deepeval/integrations/crewai/__init__.py +1 -2
- deepeval/integrations/crewai/handler.py +153 -81
- deepeval/integrations/crewai/wrapper.py +87 -0
- deepeval/integrations/pydantic_ai/instrumentator.py +48 -9
- deepeval/integrations/pydantic_ai/test_instrumentator.py +0 -0
- deepeval/metrics/faithfulness/faithfulness.py +8 -0
- deepeval/metrics/g_eval/g_eval.py +26 -15
- deepeval/metrics/prompt_alignment/prompt_alignment.py +41 -23
- deepeval/models/retry_policy.py +202 -11
- deepeval/test_run/__init__.py +2 -1
- deepeval/test_run/api.py +1 -0
- deepeval/test_run/test_run.py +85 -9
- deepeval/tracing/__init__.py +2 -0
- deepeval/tracing/otel/exporter.py +0 -6
- deepeval/tracing/otel/test_exporter.py +35 -0
- deepeval/tracing/otel/utils.py +57 -7
- deepeval/tracing/trace_context.py +14 -0
- deepeval/tracing/trace_test_manager.py +19 -0
- deepeval/tracing/tracing.py +7 -6
- deepeval/tracing/utils.py +2 -86
- deepeval/utils.py +149 -1
- {deepeval-3.6.4.dist-info → deepeval-3.6.6.dist-info}/METADATA +1 -1
- {deepeval-3.6.4.dist-info → deepeval-3.6.6.dist-info}/RECORD +35 -31
- deepeval/integrations/crewai/agent.py +0 -98
- deepeval/integrations/crewai/patch.py +0 -41
- {deepeval-3.6.4.dist-info → deepeval-3.6.6.dist-info}/LICENSE.md +0 -0
- {deepeval-3.6.4.dist-info → deepeval-3.6.6.dist-info}/WHEEL +0 -0
- {deepeval-3.6.4.dist-info → deepeval-3.6.6.dist-info}/entry_points.txt +0 -0
|
@@ -1,5 +1,5 @@
|
|
|
1
|
-
deepeval/__init__.py,sha256=
|
|
2
|
-
deepeval/_version.py,sha256=
|
|
1
|
+
deepeval/__init__.py,sha256=IqShG98ALpA1gm_qL2Jq56AJoafAHpcUTSvpgH4HpZM,3062
|
|
2
|
+
deepeval/_version.py,sha256=yb70ATorTjss8Uu310wa6TkPe0yTadiC7Lxb0-KZxMA,27
|
|
3
3
|
deepeval/annotation/__init__.py,sha256=ZFhUVNNuH_YgQSZJ-m5E9iUb9TkAkEV33a6ouMDZ8EI,111
|
|
4
4
|
deepeval/annotation/annotation.py,sha256=3j3-syeJepAcEj3u3e4T_BeRDzNr7yXGDIoNQGMKpwQ,2298
|
|
5
5
|
deepeval/annotation/api.py,sha256=EYN33ACVzVxsFleRYm60KB4Exvff3rPJKt1VBuuX970,2147
|
|
@@ -141,14 +141,15 @@ deepeval/confident/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVG
|
|
|
141
141
|
deepeval/confident/api.py,sha256=2ZhrQOtfxcnQSyY6OxrjY17y1yn-NB7pfIiJa20B1Pk,8519
|
|
142
142
|
deepeval/confident/types.py,sha256=-slFhDof_1maMgpLxqDRZv6kz6ZVY2hP_0uj_aveJKU,533
|
|
143
143
|
deepeval/config/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
144
|
-
deepeval/config/
|
|
144
|
+
deepeval/config/logging.py,sha256=ivqmhOSB-oHOOU3MvnhImrZwkkxzxKJgoKxesnWfHjg,1314
|
|
145
|
+
deepeval/config/settings.py,sha256=Ifw7HDSTaYCFk3zGHRf15uMCDZuy3NXAuNtWm4jcIUA,27575
|
|
145
146
|
deepeval/config/settings_manager.py,sha256=enahSZN8krRu7-L94OBCt99fwUIqQtMRL97PlzsuKEY,4021
|
|
146
147
|
deepeval/config/utils.py,sha256=gSOVv18Tx1R72GucbdQesbZLFL-Y9EzbS4p7qd2w_xE,3799
|
|
147
148
|
deepeval/constants.py,sha256=Qe-es-WDPJndgBspEQXxddDCVanrAu03YWCpXsUkdo0,1368
|
|
148
149
|
deepeval/contextvars.py,sha256=oqXtuYiKd4Zvc1rNoR1gcRBxzZYCGTMVn7XostwvkRI,524
|
|
149
150
|
deepeval/dataset/__init__.py,sha256=N2c-rkuxWYiiJSOZArw0H02Cwo7cnfzFuNYJlvsIBEg,249
|
|
150
151
|
deepeval/dataset/api.py,sha256=ZxkEqAF4nZH_Ys_1f5r9N2LFI_vBcAJxt8eJm7Mplpw,831
|
|
151
|
-
deepeval/dataset/dataset.py,sha256=
|
|
152
|
+
deepeval/dataset/dataset.py,sha256=giVTbjdS9zFfWNCD7WC4pyneLhXlvLJcBy38LYebIYY,51343
|
|
152
153
|
deepeval/dataset/golden.py,sha256=T-rTk4Hw1tANx_Iimv977F6Y4QK3s5OIB4PecU5FJDM,2338
|
|
153
154
|
deepeval/dataset/test_run_tracer.py,sha256=5CdpDvhzkEEBRyqWi6egocaxiN6IRS3XfbACxEQZQeM,2544
|
|
154
155
|
deepeval/dataset/types.py,sha256=CWeOIBPK2WdmRUqjFa9gfN-w2da0r8Ilzl3ToDpJQoQ,558
|
|
@@ -158,15 +159,14 @@ deepeval/evaluate/__init__.py,sha256=315IaMiYEz7oJhZ4kPTBfeCNd1xF-wWVU6KOQnrKQpE
|
|
|
158
159
|
deepeval/evaluate/api.py,sha256=rkblH0ZFAAdyuF0Ymh7JE1pIJPR9yFuPrn9SQaCEQp4,435
|
|
159
160
|
deepeval/evaluate/compare.py,sha256=tdSJY4E7YJ_zO3dzvpwngZHLiUI2YQcTWJOLI83htsQ,9855
|
|
160
161
|
deepeval/evaluate/configs.py,sha256=QfWjaWNxLsgEe8-5j4PIs5WcSyEckiWt0qdpXSpl57M,928
|
|
161
|
-
deepeval/evaluate/evaluate.py,sha256=
|
|
162
|
-
deepeval/evaluate/execute.py,sha256=
|
|
163
|
-
deepeval/evaluate/types.py,sha256=
|
|
164
|
-
deepeval/evaluate/utils.py,sha256=
|
|
162
|
+
deepeval/evaluate/evaluate.py,sha256=HoEERRLj8SVCcU1r70VQdSL4LQcSc9p20OhcD1nhEuQ,10594
|
|
163
|
+
deepeval/evaluate/execute.py,sha256=vkiWaQGBAFbLIJ1tTYpGpu1YDpDSpH6o-oPftqPlNpM,88875
|
|
164
|
+
deepeval/evaluate/types.py,sha256=zsL_lNbFMG20czzRQeWNDbLSzL8Uy7IIgvILe-X0kN0,918
|
|
165
|
+
deepeval/evaluate/utils.py,sha256=oBJFcUDYmmsRvXW7rXkQy3gI1Tuu5bixgvHx0yvnw1c,23563
|
|
165
166
|
deepeval/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
166
|
-
deepeval/integrations/crewai/__init__.py,sha256=
|
|
167
|
-
deepeval/integrations/crewai/
|
|
168
|
-
deepeval/integrations/crewai/
|
|
169
|
-
deepeval/integrations/crewai/patch.py,sha256=hqKM0RF8Tr5oeTCWuXB4oIrND_gjD2PfbmZKE_ry9C8,1299
|
|
167
|
+
deepeval/integrations/crewai/__init__.py,sha256=ERepE-h-1jWu_WqH5KSmMBFaoQ11lck88ojnrBIaTKw,72
|
|
168
|
+
deepeval/integrations/crewai/handler.py,sha256=8Cd_7N0B4o4ZhAuAbTdwOvnqUt8FRsuRsz-M_yljFWI,6508
|
|
169
|
+
deepeval/integrations/crewai/wrapper.py,sha256=ogJDfiB95EzvKIuY-YYXrD8CsH3CPpcsgNx-pSkNoAE,2378
|
|
170
170
|
deepeval/integrations/hugging_face/__init__.py,sha256=MuHIf9im9Jypq4VkfLzhklxIrd7vSTGlT74iUNSPgvg,93
|
|
171
171
|
deepeval/integrations/hugging_face/callback.py,sha256=15QQEzR34Cpdp5kUp5oVA6dEsShtiMNZ03akJWAh7lo,7911
|
|
172
172
|
deepeval/integrations/hugging_face/rich_manager.py,sha256=WvFtPGpPmGeg2Ftsnojga6yvbBLiZv_tvNbnFcGb6og,3630
|
|
@@ -182,8 +182,9 @@ deepeval/integrations/llama_index/handler.py,sha256=eqI1n8E4MsvfKoFs5Zrm9IdCR7g9
|
|
|
182
182
|
deepeval/integrations/llama_index/utils.py,sha256=mxW71-3PjvBvJpLIU0kNWuTzCidy5l_-roLt8ZyWYA0,2599
|
|
183
183
|
deepeval/integrations/pydantic_ai/__init__.py,sha256=UIkXn_g6h9LTQXG1PaWu1eCFkCssIwG48WSvN46UWgU,202
|
|
184
184
|
deepeval/integrations/pydantic_ai/agent.py,sha256=4wRV25O1tC-txH2j3TNJWry6gDNBqqThj7zgFKBxJpw,606
|
|
185
|
-
deepeval/integrations/pydantic_ai/instrumentator.py,sha256=
|
|
185
|
+
deepeval/integrations/pydantic_ai/instrumentator.py,sha256=RALOL7sz-DWgUe_fd8jWqjGzKnqgVvkoMToLfuk0u10,8493
|
|
186
186
|
deepeval/integrations/pydantic_ai/otel.py,sha256=0OuIpmaMtEt1dFWFZtYAiZ9hVCWweEWr1TRHYcDb4I8,1918
|
|
187
|
+
deepeval/integrations/pydantic_ai/test_instrumentator.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
187
188
|
deepeval/key_handler.py,sha256=damdQEBLGy4IVk5DR5-E3blIZdLbcMtyeGAFn_4_SG4,6505
|
|
188
189
|
deepeval/metrics/__init__.py,sha256=nvO0Wv2JROjK1I9MDNIFUJlrRAZI2C0xbGYSBZK5q4g,4013
|
|
189
190
|
deepeval/metrics/answer_relevancy/__init__.py,sha256=WbZUpoSg2GQoqJ4VIRirVVQ1JDx5xwT-RskwqNKfWGM,46
|
|
@@ -236,11 +237,11 @@ deepeval/metrics/dag/schema.py,sha256=rJaMlXeESN0L3Igw0FTmErqjF6s8oKkUBEjhVKyBki
|
|
|
236
237
|
deepeval/metrics/dag/templates.py,sha256=7pVyE1exfpU5z2OcAF4fH4xOxxIqO7orK3SQyMqGqEQ,2090
|
|
237
238
|
deepeval/metrics/dag/utils.py,sha256=66D88fpjIUdVwZvYV8a1L9TlX1wvbCVuE6Y8BFTbpkE,6069
|
|
238
239
|
deepeval/metrics/faithfulness/__init__.py,sha256=RffAtTOSdtWO1gHVMnPI-imJahf3JENOoJRiNw-Xv4g,43
|
|
239
|
-
deepeval/metrics/faithfulness/faithfulness.py,sha256=
|
|
240
|
+
deepeval/metrics/faithfulness/faithfulness.py,sha256=KXI5VPcOsPZ2Pk1-69tR-kq4x-FYW0TxyUF4iRfD-zE,13184
|
|
240
241
|
deepeval/metrics/faithfulness/schema.py,sha256=2dU9dwwmqpGJcWvY2webERWIfH_tn02xgLghHkAY_eM,437
|
|
241
242
|
deepeval/metrics/faithfulness/template.py,sha256=RuZ0LFm4BjZ8lhVrKPgU3ecHszwkF0fe5-BxAkaP5AA,5839
|
|
242
243
|
deepeval/metrics/g_eval/__init__.py,sha256=HAhsQFVq9LIpZXPN00Jc_WrMXrh47NIT86VnUpWM4_4,102
|
|
243
|
-
deepeval/metrics/g_eval/g_eval.py,sha256=
|
|
244
|
+
deepeval/metrics/g_eval/g_eval.py,sha256=zd4_M7UaT_l1GxHrA_g9nzCl5LXH-NYzKYLEWKLhpaU,14875
|
|
244
245
|
deepeval/metrics/g_eval/schema.py,sha256=V629txuDrr_2IEKEsgJVYYZb_pkdfcltQV9ZjvxK5co,287
|
|
245
246
|
deepeval/metrics/g_eval/template.py,sha256=mHj4-mr_HQwbCjpHg7lM_6UesoSatL3g8UGGQAOdT0U,4509
|
|
246
247
|
deepeval/metrics/g_eval/utils.py,sha256=uUT86jRXVYvLDzcnZvvfWssDyGoBHb66nWcJSg4i1u4,8784
|
|
@@ -327,7 +328,7 @@ deepeval/metrics/pii_leakage/pii_leakage.py,sha256=EIQMS_hOiYhEW5x4nYJwS6AhWl9jh
|
|
|
327
328
|
deepeval/metrics/pii_leakage/schema.py,sha256=Jk9jdf4HAa76J237mnosWOCV71pBBNdLfaVhf-4dKEg,313
|
|
328
329
|
deepeval/metrics/pii_leakage/template.py,sha256=DEW21CyR2lEI1y2C_fXgZnGJlYw0fvnB-LF-HEKZnqo,2418
|
|
329
330
|
deepeval/metrics/prompt_alignment/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
330
|
-
deepeval/metrics/prompt_alignment/prompt_alignment.py,sha256=
|
|
331
|
+
deepeval/metrics/prompt_alignment/prompt_alignment.py,sha256=JW6AjOUvJssTTwOzggT5QPp4zAb5Z4e2H-xQS83Pv3M,10083
|
|
331
332
|
deepeval/metrics/prompt_alignment/schema.py,sha256=ann3tH5XfZCLhePE2UXTSK_gkF-_udP0RP_IVHW4mF0,315
|
|
332
333
|
deepeval/metrics/prompt_alignment/template.py,sha256=6-A1rWOZWOEauSsQAXC-gVF2oXkYfgV0XqiriGJJfY0,3182
|
|
333
334
|
deepeval/metrics/ragas.py,sha256=I4EdwbsRGHlSEraKFBrDGUBrzRUBqlWqJAGg0efrZ0w,17637
|
|
@@ -386,7 +387,7 @@ deepeval/models/mlllms/__init__.py,sha256=19nN6kUB5XI0nUWUQX0aD9GBUMM8WWGvsDgKju
|
|
|
386
387
|
deepeval/models/mlllms/gemini_model.py,sha256=7tHIWD4w_fBz3L7jkKWygn1QpBPk9nl2Kw-yb0Jc3PI,10167
|
|
387
388
|
deepeval/models/mlllms/ollama_model.py,sha256=_YtYtw8oIMVVI-CFsDicsdeEJUPhw_9ArPxB_1olsJA,4798
|
|
388
389
|
deepeval/models/mlllms/openai_model.py,sha256=KgvYgQwWZ1A_Gcl6-4-W7IMqbUF9K8sNY37j5Ag7kQQ,9014
|
|
389
|
-
deepeval/models/retry_policy.py,sha256=
|
|
390
|
+
deepeval/models/retry_policy.py,sha256=efMJwjQasAE_3fstfBBmxLOzxUxws26zHP5yvEBcPfQ,31458
|
|
390
391
|
deepeval/models/summac_model.py,sha256=wKeH7pWQRXrTlzlIw_r1YCb8b7jUhWq6jUz9FiNUCSg,1992
|
|
391
392
|
deepeval/models/unbias_model.py,sha256=umOMhQLTmnD7uOuhiQufEl4Wlti4q2s3EtKOpds7zhs,597
|
|
392
393
|
deepeval/models/utils.py,sha256=-3XDgg1U7PZ0jpLFiYXxqdBhp7idvlo7RPZv5SoD8lc,1130
|
|
@@ -438,13 +439,13 @@ deepeval/test_case/llm_test_case.py,sha256=L-dCvJ4pMPPavZTyN9ZKN30h351DWI_TunmXf
|
|
|
438
439
|
deepeval/test_case/mcp.py,sha256=Z625NLvz0E_UJpbyfyuAi_4nsqKH6DByBf0rfKd70xU,1879
|
|
439
440
|
deepeval/test_case/mllm_test_case.py,sha256=8a0YoE72geX_fLI6yk_cObSxCPddwW-DOb-5OPE1-W8,5414
|
|
440
441
|
deepeval/test_case/utils.py,sha256=5lT7QmhItsQHt44-qQfspuktilcrEyvl2cS0cgUJxds,809
|
|
441
|
-
deepeval/test_run/__init__.py,sha256=
|
|
442
|
-
deepeval/test_run/api.py,sha256=
|
|
442
|
+
deepeval/test_run/__init__.py,sha256=2Jdf9V2d_Wa6Fus31aUzMZ1oy3OkuhVhZhxxcGHxZaM,771
|
|
443
|
+
deepeval/test_run/api.py,sha256=s8q2wBsE8rgOn_1GU84a8aFRjyDxMm6qQJmpl5CorCM,5484
|
|
443
444
|
deepeval/test_run/cache.py,sha256=2bKXQ4GvHe2xowFGz4ekoheLno1Yv7O6PfuanFkWnA4,12786
|
|
444
445
|
deepeval/test_run/hooks.py,sha256=Qnd06bk9RJN4WmFUzJrBAi3Xj261hzyzI2iRmG8wbKw,375
|
|
445
446
|
deepeval/test_run/hyperparameters.py,sha256=f7M07w1EfT8YPtiD9xVIVYa3ZewkxewSkK7knwv0YlY,2289
|
|
446
|
-
deepeval/test_run/test_run.py,sha256=
|
|
447
|
-
deepeval/tracing/__init__.py,sha256=
|
|
447
|
+
deepeval/test_run/test_run.py,sha256=vA515OlHBljO6M2RwAEDk0gw_8-ChPU48qjSkPZv8-U,36451
|
|
448
|
+
deepeval/tracing/__init__.py,sha256=WpiWzIIAzPIk043Si1jwTwHWxqtaIzeHj_xq9CH08nk,576
|
|
448
449
|
deepeval/tracing/api.py,sha256=rq4rB5f3tfrv6l4mRJmDrwRj5CH4dyatwxhG7p8xbVk,4867
|
|
449
450
|
deepeval/tracing/context.py,sha256=rzm42zYzP7jmQJO08AV-Qmw86ik45qRfF4UQNpGcmJw,5338
|
|
450
451
|
deepeval/tracing/offline_evals/__init__.py,sha256=bEniJAl7PmS9u2ksiOTfHtlCPJ9_CJV5R6umrUOX5MM,102
|
|
@@ -453,16 +454,19 @@ deepeval/tracing/offline_evals/span.py,sha256=pXqTVXs-WnjRVpCYYEbNe0zSM6Wz9GsKHs
|
|
|
453
454
|
deepeval/tracing/offline_evals/thread.py,sha256=bcSGFcZJKnszArOLIlWvnCyt0zSmsd7Xsw5rl4RTVFg,1981
|
|
454
455
|
deepeval/tracing/offline_evals/trace.py,sha256=vTflaTKysKRiYvKA-Nx6PUJ3J6NrRLXiIdWieVcm90E,1868
|
|
455
456
|
deepeval/tracing/otel/__init__.py,sha256=HQsaF5yLPwyW5qg8AOV81_nG_7pFHnatOTHi9Wx3HEk,88
|
|
456
|
-
deepeval/tracing/otel/exporter.py,sha256=
|
|
457
|
-
deepeval/tracing/otel/
|
|
457
|
+
deepeval/tracing/otel/exporter.py,sha256=eykoPzrRn3ljVO_JKzUeXThZ5iApyImGCDgnimIoUXs,28640
|
|
458
|
+
deepeval/tracing/otel/test_exporter.py,sha256=bezihPGWJpwUEF3ZghxqhhorocVFTO2b43jRM-JHYMU,1076
|
|
459
|
+
deepeval/tracing/otel/utils.py,sha256=Zw2_PcDm3Dtds1xUZWiVwVIKd4N98bVC8OZ_pkXv7qY,17190
|
|
458
460
|
deepeval/tracing/patchers.py,sha256=DAPNkhrDtoeyJIVeQDUMhTz-xGcXu00eqjQZmov8FiU,3096
|
|
459
461
|
deepeval/tracing/perf_epoch_bridge.py,sha256=iyAPddB6Op7NpMtPHJ29lDm53Btz9yLaN6xSCfTRQm4,1825
|
|
460
|
-
deepeval/tracing/
|
|
462
|
+
deepeval/tracing/trace_context.py,sha256=jmOH3oBKz1FeNz_J61CUfkuQ3SpyFc6n7mo_HVX6JfU,352
|
|
463
|
+
deepeval/tracing/trace_test_manager.py,sha256=wt4y7EWTRc4Bw938-UFFtXHkdFFOrnx6JaIk7J5Iulw,555
|
|
464
|
+
deepeval/tracing/tracing.py,sha256=-9GE0fjtv5xKfGZHT9LLEt-38NbqkgXRp1uZ0U1W158,42907
|
|
461
465
|
deepeval/tracing/types.py,sha256=l_utWKerNlE5H3mOKpeUJLsvpP3cMyjH7HRANNgTmSQ,5306
|
|
462
|
-
deepeval/tracing/utils.py,sha256=
|
|
463
|
-
deepeval/utils.py,sha256
|
|
464
|
-
deepeval-3.6.
|
|
465
|
-
deepeval-3.6.
|
|
466
|
-
deepeval-3.6.
|
|
467
|
-
deepeval-3.6.
|
|
468
|
-
deepeval-3.6.
|
|
466
|
+
deepeval/tracing/utils.py,sha256=6SXJ7JJu-6OUziFZ_1IJppuVv7Rlq4cw3c3B7DL_eRQ,5295
|
|
467
|
+
deepeval/utils.py,sha256=J1JNzjAlmn-UsFTK8-c5bhUuk5crwFGVCrRvle-nNmA,21533
|
|
468
|
+
deepeval-3.6.6.dist-info/LICENSE.md,sha256=0ATkuLv6QgsJTBODUHC5Rak_PArA6gv2t7inJzNTP38,11352
|
|
469
|
+
deepeval-3.6.6.dist-info/METADATA,sha256=n1o8egypf7Pr3YqgzhZ5ZnO7uSaZrWy3i5x4lyKoHmA,18754
|
|
470
|
+
deepeval-3.6.6.dist-info/WHEEL,sha256=d2fvjOD7sXsVzChCqf0Ty0JbHKBaLYwDbGQDwQTnJ50,88
|
|
471
|
+
deepeval-3.6.6.dist-info/entry_points.txt,sha256=fVr8UphXTfJe9I2rObmUtfU3gkSrYeM0pLy-NbJYg10,94
|
|
472
|
+
deepeval-3.6.6.dist-info/RECORD,,
|
|
@@ -1,98 +0,0 @@
|
|
|
1
|
-
from typing import Optional, Dict, Any, List
|
|
2
|
-
import weakref
|
|
3
|
-
from deepeval.metrics import BaseMetric
|
|
4
|
-
from deepeval.telemetry import capture_tracing_integration
|
|
5
|
-
|
|
6
|
-
try:
|
|
7
|
-
from crewai.agent import Agent as CrewAIAgent
|
|
8
|
-
|
|
9
|
-
crewai_installed = True
|
|
10
|
-
except:
|
|
11
|
-
crewai_installed = False
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
def is_crewai_installed():
|
|
15
|
-
if not crewai_installed:
|
|
16
|
-
raise ImportError(
|
|
17
|
-
"CrewAI is not installed. Please install it with `pip install crewai`."
|
|
18
|
-
)
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
class AgentRegistry:
|
|
22
|
-
"""Global registry to track CrewAI agents, their metric collections, and metrics."""
|
|
23
|
-
|
|
24
|
-
def __init__(self):
|
|
25
|
-
is_crewai_installed()
|
|
26
|
-
self._agent_metric_mapping: Dict[int, str] = {}
|
|
27
|
-
self._agent_metrics_mapping: Dict[int, List[BaseMetric]] = {}
|
|
28
|
-
self._agent_instances: Dict[int, weakref.ref] = {}
|
|
29
|
-
|
|
30
|
-
def register_agent(
|
|
31
|
-
self,
|
|
32
|
-
agent: CrewAIAgent,
|
|
33
|
-
metric_collection: Optional[str] = None,
|
|
34
|
-
metrics: Optional[List[BaseMetric]] = None,
|
|
35
|
-
):
|
|
36
|
-
"""Register a CrewAI agent with its metric collection and metrics."""
|
|
37
|
-
agent_id = id(agent)
|
|
38
|
-
self._agent_metric_mapping[agent_id] = metric_collection
|
|
39
|
-
self._agent_metrics_mapping[agent_id] = metrics or []
|
|
40
|
-
self._agent_instances[agent_id] = weakref.ref(
|
|
41
|
-
agent, self._cleanup_agent
|
|
42
|
-
)
|
|
43
|
-
|
|
44
|
-
def get_metric_collection(self, agent: CrewAIAgent) -> Optional[str]:
|
|
45
|
-
"""Get the metric collection for a given agent."""
|
|
46
|
-
agent_id = id(agent)
|
|
47
|
-
return self._agent_metric_mapping.get(agent_id)
|
|
48
|
-
|
|
49
|
-
def get_metrics(self, agent: CrewAIAgent) -> List[BaseMetric]:
|
|
50
|
-
agent_id = id(agent)
|
|
51
|
-
return self._agent_metrics_mapping.get(agent_id, [])
|
|
52
|
-
|
|
53
|
-
def _cleanup_agent(self, weak_ref):
|
|
54
|
-
"""Clean up agent references when they're garbage collected."""
|
|
55
|
-
# Find and remove the agent_id for this weak reference
|
|
56
|
-
agent_id = None
|
|
57
|
-
for aid, ref in self._agent_instances.items():
|
|
58
|
-
if ref == weak_ref:
|
|
59
|
-
agent_id = aid
|
|
60
|
-
break
|
|
61
|
-
|
|
62
|
-
if agent_id:
|
|
63
|
-
del self._agent_metric_mapping[agent_id]
|
|
64
|
-
del self._agent_metrics_mapping[agent_id]
|
|
65
|
-
del self._agent_instances[agent_id]
|
|
66
|
-
|
|
67
|
-
def get_all_agents(self) -> Dict[int, Optional[str]]:
|
|
68
|
-
"""Get all registered agents and their metric collections."""
|
|
69
|
-
return self._agent_metric_mapping.copy()
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
# Global registry instance
|
|
73
|
-
agent_registry = AgentRegistry()
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
class Agent(CrewAIAgent):
|
|
77
|
-
def __init__(
|
|
78
|
-
self,
|
|
79
|
-
*args,
|
|
80
|
-
metric_collection: Optional[str] = None,
|
|
81
|
-
metrics: Optional[List[BaseMetric]] = None,
|
|
82
|
-
**kwargs
|
|
83
|
-
):
|
|
84
|
-
with capture_tracing_integration("crewai.agent.Agent"):
|
|
85
|
-
is_crewai_installed()
|
|
86
|
-
super().__init__(*args, **kwargs)
|
|
87
|
-
# Register this agent instance with its metric collection
|
|
88
|
-
agent_registry.register_agent(self, metric_collection, metrics)
|
|
89
|
-
|
|
90
|
-
@property
|
|
91
|
-
def metric_collection(self) -> Optional[str]:
|
|
92
|
-
"""Get the metric collection for this agent."""
|
|
93
|
-
return agent_registry.get_metric_collection(self)
|
|
94
|
-
|
|
95
|
-
@property
|
|
96
|
-
def metrics(self) -> List[BaseMetric]:
|
|
97
|
-
"""Get the list of metrics for this agent."""
|
|
98
|
-
return agent_registry.get_metrics(self)
|
|
@@ -1,41 +0,0 @@
|
|
|
1
|
-
from deepeval.tracing.tracing import Observer
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
from deepeval.tracing.tracing import current_span_context
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
def patch_build_context_for_task():
|
|
8
|
-
from crewai.memory.contextual.contextual_memory import ContextualMemory
|
|
9
|
-
from crewai.task import Task
|
|
10
|
-
|
|
11
|
-
original_build_context_for_task = ContextualMemory.build_context_for_task
|
|
12
|
-
|
|
13
|
-
def patched_build_context_for_task(*args, **kwargs):
|
|
14
|
-
observer_kwargs = {
|
|
15
|
-
"observe_kwargs": {
|
|
16
|
-
"span_type": "retriever",
|
|
17
|
-
"embedder": "contextual",
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
with Observer(
|
|
21
|
-
func_name="build_context_for_task",
|
|
22
|
-
span_type="retriever",
|
|
23
|
-
**observer_kwargs
|
|
24
|
-
):
|
|
25
|
-
embedding_input = "No input"
|
|
26
|
-
if isinstance(args[1], Task):
|
|
27
|
-
embedding_input = args[1].prompt()
|
|
28
|
-
|
|
29
|
-
result = original_build_context_for_task(*args, **kwargs)
|
|
30
|
-
|
|
31
|
-
retrieval_context = []
|
|
32
|
-
if isinstance(result, str):
|
|
33
|
-
retrieval_context = [result]
|
|
34
|
-
|
|
35
|
-
current_span = current_span_context.get()
|
|
36
|
-
current_span.input = embedding_input
|
|
37
|
-
current_span.retrieval_context = retrieval_context
|
|
38
|
-
|
|
39
|
-
return result
|
|
40
|
-
|
|
41
|
-
ContextualMemory.build_context_for_task = patched_build_context_for_task
|
|
File without changes
|
|
File without changes
|
|
File without changes
|