futureagi 0.6.0__tar.gz → 0.6.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {futureagi-0.6.0 → futureagi-0.6.2}/PKG-INFO +1 -1
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/datasets/dataset.py +11 -2
- {futureagi-0.6.0 → futureagi-0.6.2}/pyproject.toml +3 -2
- futureagi-0.6.0/fi/evals/__init__.py +0 -20
- futureagi-0.6.0/fi/evals/evaluator.py +0 -375
- futureagi-0.6.0/fi/evals/metrics/__init__.py +0 -17
- futureagi-0.6.0/fi/evals/metrics/aggregated_metric.py +0 -178
- futureagi-0.6.0/fi/evals/metrics/bleu.py +0 -189
- futureagi-0.6.0/fi/evals/metrics/embedding_similarity.py +0 -174
- futureagi-0.6.0/fi/evals/metrics/laveshtein.py +0 -125
- futureagi-0.6.0/fi/evals/metrics/numeric_diff.py +0 -142
- futureagi-0.6.0/fi/evals/metrics/rouge.py +0 -164
- futureagi-0.6.0/fi/evals/metrics/semantic_list_contains.py +0 -214
- futureagi-0.6.0/fi/evals/protect.py +0 -557
- futureagi-0.6.0/fi/evals/templates.py +0 -304
- futureagi-0.6.0/fi/evals/types.py +0 -116
- {futureagi-0.6.0 → futureagi-0.6.2}/LICENSE.md +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/README.md +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/__init__.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/__init__.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/apikeys.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/auth.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/types.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/client.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/datasets/__init__.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/datasets/types.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/kb/__init__.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/kb/client.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/kb/types.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/prompt/__init__.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/prompt/client.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/prompt/types.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/__init__.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/conversational_test_case.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/general.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/unified_test_case.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/utils.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/__init__.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/constants.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/errors.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/executor.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/logging.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/routes.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/types.py +0 -0
- {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/utils.py +0 -0
|
@@ -19,8 +19,12 @@ from fi.datasets.types import (
|
|
|
19
19
|
HuggingfaceDatasetConfig,
|
|
20
20
|
Row,
|
|
21
21
|
)
|
|
22
|
-
|
|
23
|
-
from fi.evals.
|
|
22
|
+
try :
|
|
23
|
+
from fi.evals.evaluator import EvalInfoResponseHandler
|
|
24
|
+
from fi.evals.templates import EvalTemplate
|
|
25
|
+
except ImportError:
|
|
26
|
+
logging.warning(f"ai-evals is not installed. Please install it to add evaluations to your dataset.")
|
|
27
|
+
|
|
24
28
|
from fi.utils.constants import (
|
|
25
29
|
DATASET_TEMP_FILE_PREFIX,
|
|
26
30
|
DATASET_TEMP_FILE_SUFFIX,
|
|
@@ -459,6 +463,11 @@ class Dataset(APIKeyAuth):
|
|
|
459
463
|
reason_column: bool = False,
|
|
460
464
|
config: Optional[Dict[str, Any]] = None,
|
|
461
465
|
) -> "Dataset":
|
|
466
|
+
try :
|
|
467
|
+
from fi.evals.evaluator import EvalInfoResponseHandler
|
|
468
|
+
from fi.evals.templates import EvalTemplate
|
|
469
|
+
except ImportError:
|
|
470
|
+
raise DatasetError("ai-evals is not installed. Please install it to add evaluations to your dataset.")
|
|
462
471
|
|
|
463
472
|
if not self.dataset_config or not self.dataset_config.id:
|
|
464
473
|
raise DatasetError("Dataset must be configured with an ID to add an evaluation.")
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "futureagi"
|
|
3
|
-
version = "0.6.
|
|
3
|
+
version = "0.6.2"
|
|
4
4
|
description = "We help GenAI teams maintain high-accuracy for their Models in production."
|
|
5
5
|
authors = ["Future AGI <no-reply@futureagi.com>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -8,7 +8,8 @@ packages = [
|
|
|
8
8
|
{ include = "fi" }
|
|
9
9
|
]
|
|
10
10
|
exclude = [
|
|
11
|
-
"fi/integrations/otel"
|
|
11
|
+
"fi/integrations/otel",
|
|
12
|
+
"fi/evals"
|
|
12
13
|
]
|
|
13
14
|
|
|
14
15
|
[tool.poetry.dependencies]
|
|
@@ -1,20 +0,0 @@
|
|
|
1
|
-
import inspect
|
|
2
|
-
|
|
3
|
-
from fi.evals.evaluator import Evaluator, evaluate, list_evaluations # noqa: F401
|
|
4
|
-
from fi.evals.protect import Protect, protect # noqa: F401
|
|
5
|
-
from fi.evals.templates import * # noqa: F403, F401
|
|
6
|
-
from fi.evals.metrics import BLEUScore, ROUGEScore, NumericDiff, LevenshteinDistance, EmbeddingSimilarity, SemanticListContains, AggregatedMetric
|
|
7
|
-
|
|
8
|
-
# Dynamically generate __all__ from imported templates
|
|
9
|
-
_globals = globals()
|
|
10
|
-
evaluation_template_names = [
|
|
11
|
-
name
|
|
12
|
-
for name, obj in _globals.items()
|
|
13
|
-
if inspect.isclass(obj) and obj.__module__ == "fi.evals.templates"
|
|
14
|
-
]
|
|
15
|
-
|
|
16
|
-
# Add the clients separately
|
|
17
|
-
client_names = ["Evaluator", "Protect", "evaluate", "protect", "list_evaluations"]
|
|
18
|
-
|
|
19
|
-
# Combine and sort for consistency
|
|
20
|
-
__all__ = sorted(evaluation_template_names + client_names)
|
|
@@ -1,375 +0,0 @@
|
|
|
1
|
-
import inspect
|
|
2
|
-
import json
|
|
3
|
-
import logging
|
|
4
|
-
from functools import lru_cache
|
|
5
|
-
from typing import Any, Dict, List, Optional, Union
|
|
6
|
-
from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
|
|
7
|
-
from collections import defaultdict
|
|
8
|
-
import os
|
|
9
|
-
|
|
10
|
-
from requests import Response
|
|
11
|
-
|
|
12
|
-
from fi.api.auth import APIKeyAuth, ResponseHandler
|
|
13
|
-
from fi.api.types import HttpMethod, RequestConfig
|
|
14
|
-
from fi.evals.templates import EvalTemplate
|
|
15
|
-
from fi.evals.types import BatchRunResult, EvalResult, EvalResultMetric
|
|
16
|
-
from fi.testcases import TestCase, MLLMImage, MLLMAudio
|
|
17
|
-
from fi.utils.errors import InvalidAuthError
|
|
18
|
-
from fi.utils.routes import Routes
|
|
19
|
-
from fi.utils.utils import get_keys_from_env, get_base_url_from_env
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
|
|
23
|
-
"""Handles responses for evaluation requests"""
|
|
24
|
-
|
|
25
|
-
@classmethod
|
|
26
|
-
def _parse_success(cls, response: Response) -> BatchRunResult:
|
|
27
|
-
return cls.convert_to_batch_results(response.json())
|
|
28
|
-
|
|
29
|
-
@classmethod
|
|
30
|
-
def _handle_error(cls, response: Response) -> None:
|
|
31
|
-
if response.status_code == 400:
|
|
32
|
-
raise Exception(
|
|
33
|
-
f"Evaluation failed with a 400 Bad Request. Please check your input data and evaluation configuration. Response: {response.text}"
|
|
34
|
-
)
|
|
35
|
-
elif response.status_code == 403:
|
|
36
|
-
raise InvalidAuthError()
|
|
37
|
-
else:
|
|
38
|
-
raise Exception(
|
|
39
|
-
f"Error in evaluation: {response.status_code}, response: {response.text}"
|
|
40
|
-
)
|
|
41
|
-
|
|
42
|
-
@classmethod
|
|
43
|
-
def convert_to_batch_results(cls, response: Dict[str, Any]) -> BatchRunResult:
|
|
44
|
-
"""
|
|
45
|
-
Convert API response to BatchRunResult
|
|
46
|
-
|
|
47
|
-
Args:
|
|
48
|
-
response: Raw API response dictionary
|
|
49
|
-
|
|
50
|
-
Returns:
|
|
51
|
-
BatchRunResult containing evaluation results
|
|
52
|
-
"""
|
|
53
|
-
eval_results = []
|
|
54
|
-
|
|
55
|
-
for result in response.get("result", {}):
|
|
56
|
-
for evaluation in result.get("evaluations", []):
|
|
57
|
-
new_metadata = {}
|
|
58
|
-
if evaluation.get("metadata"):
|
|
59
|
-
if isinstance(evaluation.get("metadata"), dict):
|
|
60
|
-
metadata = evaluation.get("metadata")
|
|
61
|
-
elif isinstance(evaluation.get("metadata"), str):
|
|
62
|
-
metadata = json.loads(evaluation.get("metadata"))
|
|
63
|
-
else:
|
|
64
|
-
metadata = {}
|
|
65
|
-
new_metadata["usage"] = metadata.get("usage", {})
|
|
66
|
-
new_metadata["cost"] = metadata.get("cost", {})
|
|
67
|
-
new_metadata["explanation"] = metadata.get("explanation", {})
|
|
68
|
-
eval_results.append(
|
|
69
|
-
EvalResult(
|
|
70
|
-
data=evaluation.get("data"),
|
|
71
|
-
failure=evaluation.get("failure"),
|
|
72
|
-
reason=evaluation.get("reason", ""),
|
|
73
|
-
runtime=evaluation.get("runtime", 0),
|
|
74
|
-
metadata=new_metadata,
|
|
75
|
-
metrics=[
|
|
76
|
-
EvalResultMetric(id=metric["id"], value=metric["value"])
|
|
77
|
-
for metric in evaluation.get("metrics", [])
|
|
78
|
-
],
|
|
79
|
-
)
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
return BatchRunResult(eval_results=eval_results)
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
class EvalInfoResponseHandler(ResponseHandler[dict, None]):
|
|
86
|
-
"""Handles responses for evaluation info requests"""
|
|
87
|
-
|
|
88
|
-
@classmethod
|
|
89
|
-
def _parse_success(cls, response: Response) -> dict:
|
|
90
|
-
data = response.json()
|
|
91
|
-
if "result" in data:
|
|
92
|
-
return data["result"]
|
|
93
|
-
else:
|
|
94
|
-
raise Exception(f"Failed to get evaluation info: {data}")
|
|
95
|
-
|
|
96
|
-
@classmethod
|
|
97
|
-
def _handle_error(cls, response: Response) -> None:
|
|
98
|
-
if response.status_code == 400:
|
|
99
|
-
response.raise_for_status()
|
|
100
|
-
if response.status_code == 403:
|
|
101
|
-
raise InvalidAuthError()
|
|
102
|
-
raise Exception(f"Failed to get evaluation info: {response.status_code}")
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
class Evaluator(APIKeyAuth):
|
|
106
|
-
"""Client for evaluating LLM test cases"""
|
|
107
|
-
|
|
108
|
-
def __init__(
|
|
109
|
-
self,
|
|
110
|
-
fi_api_key: Optional[str] = None,
|
|
111
|
-
fi_secret_key: Optional[str] = None,
|
|
112
|
-
fi_base_url: Optional[str] = None,
|
|
113
|
-
**kwargs,
|
|
114
|
-
) -> None:
|
|
115
|
-
"""
|
|
116
|
-
Initialize the Eval Client
|
|
117
|
-
|
|
118
|
-
Args:
|
|
119
|
-
fi_api_key: API key
|
|
120
|
-
fi_secret_key: Secret key
|
|
121
|
-
fi_base_url: Base URL
|
|
122
|
-
|
|
123
|
-
Keyword Args:
|
|
124
|
-
timeout: Optional timeout value in seconds (default: 200)
|
|
125
|
-
max_queue_bound: Optional maximum queue size (default: 5000)
|
|
126
|
-
max_workers: Optional maximum number of workers (default: 8)
|
|
127
|
-
"""
|
|
128
|
-
super().__init__(fi_api_key, fi_secret_key, fi_base_url, **kwargs)
|
|
129
|
-
self._max_workers = kwargs.get("max_workers", 8) # Default to 8 if not provided
|
|
130
|
-
|
|
131
|
-
def evaluate(
|
|
132
|
-
self,
|
|
133
|
-
eval_templates: Union[str, type[EvalTemplate]],
|
|
134
|
-
inputs: Union[
|
|
135
|
-
TestCase,
|
|
136
|
-
List[TestCase],
|
|
137
|
-
Dict[str, Any],
|
|
138
|
-
List[Dict[str, Any]],
|
|
139
|
-
],
|
|
140
|
-
timeout: Optional[int] = None,
|
|
141
|
-
model_name: Optional[str] = None,
|
|
142
|
-
) -> BatchRunResult:
|
|
143
|
-
"""
|
|
144
|
-
Run a single or batch of evaluations independently
|
|
145
|
-
|
|
146
|
-
Args:
|
|
147
|
-
eval_templates: Evaluation name string (e.g., "Factual Accuracy")
|
|
148
|
-
inputs: Single test case or list of test cases
|
|
149
|
-
timeout: Optional timeout value for the evaluation
|
|
150
|
-
model_name: Optional model name to use for the evaluation for Future AGI Agents
|
|
151
|
-
|
|
152
|
-
Returns:
|
|
153
|
-
BatchRunResult containing evaluation results
|
|
154
|
-
|
|
155
|
-
Raises:
|
|
156
|
-
ValidationError: If the inputs do not match the evaluation templates
|
|
157
|
-
Exception: If the API request fails
|
|
158
|
-
"""
|
|
159
|
-
if not isinstance(inputs, list):
|
|
160
|
-
inputs = [inputs]
|
|
161
|
-
|
|
162
|
-
def _extract_name(t) -> str | None:
|
|
163
|
-
if isinstance(t, str):
|
|
164
|
-
return t
|
|
165
|
-
if isinstance(t, EvalTemplate):
|
|
166
|
-
return t.eval_name
|
|
167
|
-
if inspect.isclass(t) and issubclass(t, EvalTemplate):
|
|
168
|
-
return t.eval_name
|
|
169
|
-
return None
|
|
170
|
-
eval_name = _extract_name(
|
|
171
|
-
eval_templates[0] if isinstance(eval_templates, list) else eval_templates
|
|
172
|
-
)
|
|
173
|
-
if eval_name is None:
|
|
174
|
-
raise TypeError(
|
|
175
|
-
"Unsupported eval_templates argument. "
|
|
176
|
-
"Expect eval template class/obj or name str."
|
|
177
|
-
)
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
transformed_api_inputs = defaultdict(list)
|
|
182
|
-
|
|
183
|
-
if not inputs:
|
|
184
|
-
# Allows an empty transformed_api_inputs if the original inputs list was empty.
|
|
185
|
-
pass
|
|
186
|
-
else:
|
|
187
|
-
for tc in inputs:
|
|
188
|
-
dumped_tc: Dict[str, Any]
|
|
189
|
-
# Handle both TestCase objects and raw dictionaries
|
|
190
|
-
if isinstance(tc, TestCase):
|
|
191
|
-
# TestCase object
|
|
192
|
-
dumped_tc = tc.model_dump(exclude_none=True, by_alias=False)
|
|
193
|
-
elif isinstance(tc, dict):
|
|
194
|
-
# Raw dictionary
|
|
195
|
-
processed_tc = {}
|
|
196
|
-
for key, value in tc.items():
|
|
197
|
-
if isinstance(value, str):
|
|
198
|
-
original_value = value # Keep a copy for fallback
|
|
199
|
-
try:
|
|
200
|
-
mllm_image = MLLMImage(url=value)
|
|
201
|
-
processed_tc[key] = mllm_image.url
|
|
202
|
-
logging.debug(f"Processed key '{key}' as MLLMImage.")
|
|
203
|
-
continue
|
|
204
|
-
except ValueError as e_img:
|
|
205
|
-
logging.debug(f"Key '{key}' not processed as MLLMImage (Error: {e_img}). Trying MLLMAudio.")
|
|
206
|
-
try:
|
|
207
|
-
mllm_audio = MLLMAudio(url=value)
|
|
208
|
-
if not mllm_audio.is_plain_text:
|
|
209
|
-
processed_tc[key] = mllm_audio.url
|
|
210
|
-
logging.debug(f"Processed key '{key}' as MLLMAudio.")
|
|
211
|
-
continue
|
|
212
|
-
else:
|
|
213
|
-
logging.debug(f"Key '{key}' treated as plain text by MLLMAudio. Using original value.")
|
|
214
|
-
processed_tc[key] = original_value
|
|
215
|
-
except ValueError as e_audio:
|
|
216
|
-
logging.debug(f"Key '{key}' not processed as MLLMAudio (Error: {e_audio}). Using original value.")
|
|
217
|
-
processed_tc[key] = original_value
|
|
218
|
-
except Exception as e_gen_audio:
|
|
219
|
-
logging.warning(f"Unexpected error processing key '{key}' as MLLMAudio (Error: {e_gen_audio}). Using original value.")
|
|
220
|
-
processed_tc[key] = original_value
|
|
221
|
-
except Exception as e_gen_image:
|
|
222
|
-
logging.warning(f"Unexpected error processing key '{key}' as MLLMImage (Error: {e_gen_image}). Using original value.")
|
|
223
|
-
processed_tc[key] = original_value
|
|
224
|
-
else:
|
|
225
|
-
# Not a string, keep original value
|
|
226
|
-
processed_tc[key] = value
|
|
227
|
-
dumped_tc = processed_tc
|
|
228
|
-
else:
|
|
229
|
-
raise TypeError(
|
|
230
|
-
f"Invalid input type: {type(tc)}. Each input must be a TestCase object or a dictionary with valid keys."
|
|
231
|
-
)
|
|
232
|
-
|
|
233
|
-
for key, value in dumped_tc.items():
|
|
234
|
-
transformed_api_inputs[key].append(value)
|
|
235
|
-
|
|
236
|
-
final_api_payload = {
|
|
237
|
-
"eval_name": eval_name,
|
|
238
|
-
"inputs": dict(transformed_api_inputs),
|
|
239
|
-
"model": model_name,
|
|
240
|
-
}
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
all_results = []
|
|
244
|
-
failed_inputs = []
|
|
245
|
-
with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
|
|
246
|
-
# Submit the batch only once
|
|
247
|
-
future = executor.submit(
|
|
248
|
-
self.request,
|
|
249
|
-
config=RequestConfig(
|
|
250
|
-
method=HttpMethod.POST,
|
|
251
|
-
url=f"{self._base_url}/{Routes.evaluatev2.value}",
|
|
252
|
-
json=final_api_payload,
|
|
253
|
-
timeout=timeout or self._default_timeout,
|
|
254
|
-
),
|
|
255
|
-
response_handler=EvalResponseHandler,
|
|
256
|
-
)
|
|
257
|
-
future_to_input = {future: inputs} # map single future to all inputs
|
|
258
|
-
|
|
259
|
-
for future in as_completed(future_to_input):
|
|
260
|
-
try:
|
|
261
|
-
response: BatchRunResult = future.result(timeout=timeout or self._default_timeout)
|
|
262
|
-
all_results.extend(response.eval_results)
|
|
263
|
-
except TimeoutError:
|
|
264
|
-
input_case = future_to_input[future]
|
|
265
|
-
logging.error(f"Evaluation timed out for input: {input_case}")
|
|
266
|
-
failed_inputs.append(input_case)
|
|
267
|
-
|
|
268
|
-
except Exception as exc:
|
|
269
|
-
input_case = future_to_input[future]
|
|
270
|
-
logging.error(f"Evaluation failed for input {input_case}: {str(exc)}")
|
|
271
|
-
failed_inputs.append(input_case)
|
|
272
|
-
|
|
273
|
-
if failed_inputs:
|
|
274
|
-
logging.warning(f"Failed to evaluate {len(failed_inputs)} inputs out of {len(inputs)} total inputs")
|
|
275
|
-
|
|
276
|
-
return BatchRunResult(eval_results=all_results)
|
|
277
|
-
|
|
278
|
-
def _validate_inputs(
|
|
279
|
-
self,
|
|
280
|
-
inputs: List[TestCase],
|
|
281
|
-
eval_objects: List[EvalTemplate],
|
|
282
|
-
):
|
|
283
|
-
"""
|
|
284
|
-
Validate the inputs against the evaluation templates
|
|
285
|
-
|
|
286
|
-
Args:
|
|
287
|
-
inputs: List of test cases to validate
|
|
288
|
-
eval_objects: List of evaluation templates to validate against
|
|
289
|
-
|
|
290
|
-
Returns:
|
|
291
|
-
bool: True if validation passes
|
|
292
|
-
|
|
293
|
-
Raises:
|
|
294
|
-
Exception: If validation fails or templates don't share common tags
|
|
295
|
-
"""
|
|
296
|
-
|
|
297
|
-
# First validate that all eval objects share at least one common tag
|
|
298
|
-
if len(eval_objects) > 1:
|
|
299
|
-
# Get sets of tags for each eval object
|
|
300
|
-
tag_sets = [set(obj.eval_tags) for obj in eval_objects]
|
|
301
|
-
|
|
302
|
-
# Find intersection of all tag sets
|
|
303
|
-
common_tags = set.intersection(*tag_sets)
|
|
304
|
-
|
|
305
|
-
if not common_tags:
|
|
306
|
-
template_names = [obj.name for obj in eval_objects]
|
|
307
|
-
raise Exception(
|
|
308
|
-
f"Evaluation templates {template_names} must share at least one common tag. "
|
|
309
|
-
f"Current tags for each template: {[list(tags) for tags in tag_sets]}"
|
|
310
|
-
)
|
|
311
|
-
|
|
312
|
-
# Then validate each eval object's required inputs
|
|
313
|
-
for eval_object in eval_objects:
|
|
314
|
-
eval_object.validate_input(inputs)
|
|
315
|
-
|
|
316
|
-
return True
|
|
317
|
-
|
|
318
|
-
def _get_eval_configs(
|
|
319
|
-
self, eval_templates: Union[str, List[str]]
|
|
320
|
-
) -> List[EvalTemplate]:
|
|
321
|
-
if isinstance(eval_templates, str):
|
|
322
|
-
eval_templates = [eval_templates]
|
|
323
|
-
|
|
324
|
-
for template in eval_templates:
|
|
325
|
-
eval_info = self._get_eval_info(template)
|
|
326
|
-
template.eval_id = eval_info["eval_id"]
|
|
327
|
-
template.name = eval_info["name"]
|
|
328
|
-
template.description = eval_info["description"]
|
|
329
|
-
template.eval_tags = eval_info["eval_tags"]
|
|
330
|
-
template.required_keys = eval_info["config"]["required_keys"]
|
|
331
|
-
template.output = eval_info["config"]["output"]
|
|
332
|
-
template.eval_type_id = eval_info["config"]["eval_type_id"]
|
|
333
|
-
template.config_schema = (
|
|
334
|
-
eval_info["config"]["config"] if "config" in eval_info["config"] else {}
|
|
335
|
-
)
|
|
336
|
-
template.criteria = eval_info["criteria"]
|
|
337
|
-
template.choices = eval_info["choices"]
|
|
338
|
-
template.multi_choice = eval_info["multi_choice"]
|
|
339
|
-
return eval_templates
|
|
340
|
-
|
|
341
|
-
@lru_cache(maxsize=100)
|
|
342
|
-
def _get_eval_info(self, eval_name: str) -> Dict[str, Any]:
|
|
343
|
-
url = (
|
|
344
|
-
self._base_url
|
|
345
|
-
+ "/"
|
|
346
|
-
+ Routes.get_eval_templates.value
|
|
347
|
-
)
|
|
348
|
-
response = self.request(
|
|
349
|
-
config=RequestConfig(method=HttpMethod.GET, url=url),
|
|
350
|
-
response_handler=EvalInfoResponseHandler,
|
|
351
|
-
)
|
|
352
|
-
eval_info = next((item for item in response if item["name"] == eval_name), None)
|
|
353
|
-
if eval_info is None:
|
|
354
|
-
raise KeyError(f"Evaluation template '{eval_name}' not found in registry")
|
|
355
|
-
if not eval_info:
|
|
356
|
-
raise Exception(f"Evaluation template with name '{eval_name}' not found")
|
|
357
|
-
return eval_info
|
|
358
|
-
|
|
359
|
-
def list_evaluations(self):
|
|
360
|
-
"""
|
|
361
|
-
Fetch information about all available evaluation templates by getting eval_info
|
|
362
|
-
for each template class defined in templates.py.
|
|
363
|
-
|
|
364
|
-
Returns:
|
|
365
|
-
List[Dict[str, Any]]: List of evaluation template information dictionaries
|
|
366
|
-
"""
|
|
367
|
-
config = RequestConfig(method=HttpMethod.GET,
|
|
368
|
-
url=f"{self._base_url}/{Routes.get_eval_templates.value}")
|
|
369
|
-
|
|
370
|
-
response = self.request(config=config, response_handler=EvalInfoResponseHandler)
|
|
371
|
-
|
|
372
|
-
return response
|
|
373
|
-
evaluate = lambda eval_templates, inputs, timeout=None: Evaluator().evaluate(eval_templates, inputs, timeout)
|
|
374
|
-
|
|
375
|
-
list_evaluations = lambda: Evaluator().list_evaluations()
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
from fi.evals.metrics.bleu import BLEUScore
|
|
2
|
-
from fi.evals.metrics.rouge import ROUGEScore
|
|
3
|
-
from fi.evals.metrics.numeric_diff import NumericDiff
|
|
4
|
-
from fi.evals.metrics.laveshtein import LevenshteinDistance
|
|
5
|
-
from fi.evals.metrics.embedding_similarity import EmbeddingSimilarity
|
|
6
|
-
from fi.evals.metrics.semantic_list_contains import SemanticListContains
|
|
7
|
-
from fi.evals.metrics.aggregated_metric import AggregatedMetric
|
|
8
|
-
|
|
9
|
-
__all__ = [
|
|
10
|
-
"BLEUScore",
|
|
11
|
-
"ROUGEScore",
|
|
12
|
-
"NumericDiff",
|
|
13
|
-
"LevenshteinDistance",
|
|
14
|
-
"EmbeddingSimilarity",
|
|
15
|
-
"SemanticListContains",
|
|
16
|
-
"AggregatedMetric"
|
|
17
|
-
]
|
|
@@ -1,178 +0,0 @@
|
|
|
1
|
-
import string
|
|
2
|
-
from typing import List, Union, Any, Dict, Optional, Tuple
|
|
3
|
-
import json
|
|
4
|
-
|
|
5
|
-
from fi.evals.templates import EvalTemplate
|
|
6
|
-
from fi.evals.types import EvalResult, EvalResultMetric, BatchRunResult
|
|
7
|
-
from fi.testcases import TestCase
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
class AggregatedMetric(EvalTemplate):
|
|
11
|
-
"""
|
|
12
|
-
AggregatedMetric combines multiple metric evaluators into a single aggregated score.
|
|
13
|
-
|
|
14
|
-
Possible values for aggregator:
|
|
15
|
-
- 'average': Computes the mean of all metric scores.
|
|
16
|
-
- 'weighted_average': Computes the weighted mean using provided weights.
|
|
17
|
-
|
|
18
|
-
Example config:
|
|
19
|
-
config = {
|
|
20
|
-
"aggregator": "average", # or "weighted_average"
|
|
21
|
-
"weights": [0.5, 0.5], # Only for weighted_average
|
|
22
|
-
"metrics": [metric1, metric2]
|
|
23
|
-
}
|
|
24
|
-
|
|
25
|
-
Example usage:
|
|
26
|
-
agg_metric = AggregatedMetric(config)
|
|
27
|
-
agg_metric.validate_input(test_cases)
|
|
28
|
-
result = agg_metric.evaluate(test_cases)
|
|
29
|
-
"""
|
|
30
|
-
EVAL_ID = "82"
|
|
31
|
-
NAME = "Aggregated Metric"
|
|
32
|
-
DESCRIPTION = "Combines multiple metrics into a single aggregated score"
|
|
33
|
-
EVAL_TAGS = ["metric", "aggregation", "composite", "multi-metric"]
|
|
34
|
-
REQUIRED_KEYS = ["response", "expected_text"]
|
|
35
|
-
OUTPUT = "score"
|
|
36
|
-
EVAL_TYPE_ID = "metric"
|
|
37
|
-
DEFAULT_CONFIG_SCHEMA = {
|
|
38
|
-
"aggregator": "average",
|
|
39
|
-
"weights": None,
|
|
40
|
-
"metrics": []
|
|
41
|
-
}
|
|
42
|
-
CRITERIA = "Aggregates multiple evaluation metrics into a combined score"
|
|
43
|
-
CHOICES = []
|
|
44
|
-
MULTI_CHOICE = False
|
|
45
|
-
SUPPORTED_AGGREGATORS = ["average", "weighted_average"]
|
|
46
|
-
|
|
47
|
-
def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
|
|
48
|
-
config = config or {}
|
|
49
|
-
self.aggregator = config.get("aggregator", self.DEFAULT_CONFIG_SCHEMA["aggregator"])
|
|
50
|
-
self.weights = config.get("weights", self.DEFAULT_CONFIG_SCHEMA["weights"])
|
|
51
|
-
self.metrics = config.get("metrics", self.DEFAULT_CONFIG_SCHEMA["metrics"])
|
|
52
|
-
|
|
53
|
-
# Set required instance attributes for EvalTemplate
|
|
54
|
-
self.name = self.NAME
|
|
55
|
-
self.description = self.DESCRIPTION
|
|
56
|
-
self.eval_id = self.EVAL_ID
|
|
57
|
-
self.eval_tags = self.EVAL_TAGS
|
|
58
|
-
self.required_keys = self.REQUIRED_KEYS
|
|
59
|
-
self.output = self.OUTPUT
|
|
60
|
-
self.eval_type_id = self.EVAL_TYPE_ID
|
|
61
|
-
self.config_schema = self.DEFAULT_CONFIG_SCHEMA
|
|
62
|
-
self.criteria = self.CRITERIA
|
|
63
|
-
self.choices = self.CHOICES
|
|
64
|
-
self.multi_choice = self.MULTI_CHOICE
|
|
65
|
-
|
|
66
|
-
if not self.metrics:
|
|
67
|
-
raise ValueError("AggregatedMetric requires at least one metric evaluator")
|
|
68
|
-
|
|
69
|
-
if self.aggregator not in self.SUPPORTED_AGGREGATORS:
|
|
70
|
-
raise ValueError(f"Unsupported aggregator: {self.aggregator}")
|
|
71
|
-
if self.aggregator == "weighted_average":
|
|
72
|
-
if self.weights is None or len(self.weights) != len(self.metrics):
|
|
73
|
-
raise ValueError("Weights required for weighted_average and must match the number of metrics")
|
|
74
|
-
|
|
75
|
-
self.metric_names = [getattr(m, 'NAME', getattr(m, 'name', str(m))) for m in self.metrics]
|
|
76
|
-
|
|
77
|
-
super().__init__(config)
|
|
78
|
-
|
|
79
|
-
def validate_input(self, inputs: List[TestCase]):
|
|
80
|
-
for test_case in inputs:
|
|
81
|
-
if not hasattr(test_case, "response") or test_case.response is None:
|
|
82
|
-
raise ValueError("TestCase must have a 'response' field for AggregatedMetric evaluation")
|
|
83
|
-
if not hasattr(test_case, "expected_text") or test_case.expected_text is None:
|
|
84
|
-
raise ValueError("TestCase must have an 'expected_text' field for AggregatedMetric evaluation")
|
|
85
|
-
for metric in self.metrics:
|
|
86
|
-
if hasattr(metric, 'validate_input'):
|
|
87
|
-
metric.validate_input(inputs)
|
|
88
|
-
return True
|
|
89
|
-
|
|
90
|
-
def evaluate(self, inputs: List[TestCase]) -> BatchRunResult:
|
|
91
|
-
try:
|
|
92
|
-
self.validate_input(inputs)
|
|
93
|
-
except Exception as e:
|
|
94
|
-
eval_result = EvalResult(
|
|
95
|
-
data=[0.0, {}],
|
|
96
|
-
failure=True,
|
|
97
|
-
reason=f"Input validation failed: {str(e)}",
|
|
98
|
-
runtime=0,
|
|
99
|
-
metadata={
|
|
100
|
-
"aggregator": self.aggregator,
|
|
101
|
-
"metrics": self.metric_names,
|
|
102
|
-
"weights": self.weights,
|
|
103
|
-
"individual_scores": {}
|
|
104
|
-
},
|
|
105
|
-
metrics=[
|
|
106
|
-
EvalResultMetric(
|
|
107
|
-
id="aggregated_metric_score",
|
|
108
|
-
value=0.0
|
|
109
|
-
)
|
|
110
|
-
]
|
|
111
|
-
)
|
|
112
|
-
return BatchRunResult(eval_results=[eval_result])
|
|
113
|
-
|
|
114
|
-
eval_results = []
|
|
115
|
-
for test_case in inputs:
|
|
116
|
-
try:
|
|
117
|
-
metric_results = []
|
|
118
|
-
for metric in self.metrics:
|
|
119
|
-
try:
|
|
120
|
-
if hasattr(metric, 'evaluate'):
|
|
121
|
-
result = metric.evaluate([test_case])
|
|
122
|
-
if result.eval_results and result.eval_results[0].metrics:
|
|
123
|
-
metric_value = result.eval_results[0].metrics[0].value
|
|
124
|
-
metric_results.append(self._normalize_score(metric_value))
|
|
125
|
-
else:
|
|
126
|
-
metric_results.append(0.0)
|
|
127
|
-
else:
|
|
128
|
-
metric_results.append(0.0)
|
|
129
|
-
except Exception as metric_exc:
|
|
130
|
-
metric_results.append(0.0)
|
|
131
|
-
if self.aggregator == "average":
|
|
132
|
-
aggregated_score = sum(metric_results) / len(metric_results) if metric_results else 0.0
|
|
133
|
-
elif self.aggregator == "weighted_average":
|
|
134
|
-
weighted_sum = sum(w * s for w, s in zip(self.weights, metric_results))
|
|
135
|
-
total_weight = sum(self.weights)
|
|
136
|
-
aggregated_score = weighted_sum / total_weight if total_weight > 0 else 0.0
|
|
137
|
-
else:
|
|
138
|
-
aggregated_score = 0.0
|
|
139
|
-
metric_details = {
|
|
140
|
-
name: score for name, score in zip(self.metric_names, metric_results)
|
|
141
|
-
}
|
|
142
|
-
failure = False
|
|
143
|
-
reason = ""
|
|
144
|
-
except Exception as e:
|
|
145
|
-
aggregated_score = 0.0
|
|
146
|
-
metric_details = {}
|
|
147
|
-
failure = True
|
|
148
|
-
reason = f"Evaluation failed: {str(e)}"
|
|
149
|
-
eval_result = EvalResult(
|
|
150
|
-
data=[aggregated_score, metric_details],
|
|
151
|
-
failure=failure,
|
|
152
|
-
reason=reason,
|
|
153
|
-
runtime=0,
|
|
154
|
-
metadata={
|
|
155
|
-
"aggregator": self.aggregator,
|
|
156
|
-
"metrics": self.metric_names,
|
|
157
|
-
"weights": self.weights,
|
|
158
|
-
"individual_scores": metric_details
|
|
159
|
-
},
|
|
160
|
-
metrics=[
|
|
161
|
-
EvalResultMetric(
|
|
162
|
-
id="aggregated_metric_score",
|
|
163
|
-
value=aggregated_score
|
|
164
|
-
)
|
|
165
|
-
]
|
|
166
|
-
)
|
|
167
|
-
eval_results.append(eval_result)
|
|
168
|
-
return BatchRunResult(eval_results=eval_results)
|
|
169
|
-
|
|
170
|
-
def _normalize_score(self, value: Any) -> float:
|
|
171
|
-
"""Convert various score types to a float between 0 and 1"""
|
|
172
|
-
if isinstance(value, bool):
|
|
173
|
-
return 1.0 if value else 0.0
|
|
174
|
-
try:
|
|
175
|
-
float_value = float(value)
|
|
176
|
-
return max(0.0, min(1.0, float_value))
|
|
177
|
-
except (ValueError, TypeError):
|
|
178
|
-
return 0.0
|