futureagi 0.3.6__tar.gz → 0.3.7__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {futureagi-0.3.6 → futureagi-0.3.7}/PKG-INFO +1 -1
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/evals/evaluator.py +24 -6
- {futureagi-0.3.6 → futureagi-0.3.7}/pyproject.toml +1 -1
- {futureagi-0.3.6 → futureagi-0.3.7}/LICENSE.md +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/README.md +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/api/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/api/auth.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/api/types.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/client.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/datasets/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/datasets/client.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/datasets/types.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/evals/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/evals/templates.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/evals/types.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/integrations/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/integrations/providers/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/integrations/providers/types.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/testcases/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/testcases/conversational_test_case.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/testcases/general.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/testcases/utils.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/__init__.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/constants.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/errors.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/executor.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/logging.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/routes.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/types.py +0 -0
- {futureagi-0.3.6 → futureagi-0.3.7}/fi/utils/utils.py +0 -0
|
@@ -64,6 +64,27 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
|
|
|
64
64
|
return BatchRunResult(eval_results=eval_results)
|
|
65
65
|
|
|
66
66
|
|
|
67
|
+
class EvalInfoResponseHandler(ResponseHandler[dict, None]):
|
|
68
|
+
"""Handles responses for evaluation info requests"""
|
|
69
|
+
|
|
70
|
+
@classmethod
|
|
71
|
+
def _parse_success(cls, response: Response) -> dict:
|
|
72
|
+
data = response.json()
|
|
73
|
+
if "result" in data:
|
|
74
|
+
print("result", data["result"])
|
|
75
|
+
return data["result"]
|
|
76
|
+
else:
|
|
77
|
+
raise Exception(f"Failed to get evaluation info: {data}")
|
|
78
|
+
|
|
79
|
+
@classmethod
|
|
80
|
+
def _handle_error(cls, response: Response) -> None:
|
|
81
|
+
if response.status_code == 400:
|
|
82
|
+
response.raise_for_status()
|
|
83
|
+
if response.status_code == 403:
|
|
84
|
+
raise InvalidAuthError()
|
|
85
|
+
raise Exception(f"Failed to get evaluation info: {response.status_code}")
|
|
86
|
+
|
|
87
|
+
|
|
67
88
|
class EvalClient(APIKeyAuth):
|
|
68
89
|
"""Client for evaluating LLM test cases"""
|
|
69
90
|
|
|
@@ -199,7 +220,6 @@ class EvalClient(APIKeyAuth):
|
|
|
199
220
|
|
|
200
221
|
for template in eval_templates:
|
|
201
222
|
eval_id = template.eval_id
|
|
202
|
-
|
|
203
223
|
eval_info = self._get_eval_info(eval_id)
|
|
204
224
|
template.name = eval_info["name"]
|
|
205
225
|
template.description = eval_info["description"]
|
|
@@ -223,9 +243,7 @@ class EvalClient(APIKeyAuth):
|
|
|
223
243
|
+ Routes.evaluate_template.value.format(eval_id=eval_id)
|
|
224
244
|
)
|
|
225
245
|
response = self.request(
|
|
226
|
-
config=RequestConfig(
|
|
227
|
-
|
|
228
|
-
url=url,
|
|
229
|
-
),
|
|
246
|
+
config=RequestConfig(method=HttpMethod.GET, url=url),
|
|
247
|
+
response_handler=EvalInfoResponseHandler,
|
|
230
248
|
)
|
|
231
|
-
return response
|
|
249
|
+
return response
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|