futureagi 0.6.0__tar.gz → 0.6.2__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. {futureagi-0.6.0 → futureagi-0.6.2}/PKG-INFO +1 -1
  2. {futureagi-0.6.0 → futureagi-0.6.2}/fi/datasets/dataset.py +11 -2
  3. {futureagi-0.6.0 → futureagi-0.6.2}/pyproject.toml +3 -2
  4. futureagi-0.6.0/fi/evals/__init__.py +0 -20
  5. futureagi-0.6.0/fi/evals/evaluator.py +0 -375
  6. futureagi-0.6.0/fi/evals/metrics/__init__.py +0 -17
  7. futureagi-0.6.0/fi/evals/metrics/aggregated_metric.py +0 -178
  8. futureagi-0.6.0/fi/evals/metrics/bleu.py +0 -189
  9. futureagi-0.6.0/fi/evals/metrics/embedding_similarity.py +0 -174
  10. futureagi-0.6.0/fi/evals/metrics/laveshtein.py +0 -125
  11. futureagi-0.6.0/fi/evals/metrics/numeric_diff.py +0 -142
  12. futureagi-0.6.0/fi/evals/metrics/rouge.py +0 -164
  13. futureagi-0.6.0/fi/evals/metrics/semantic_list_contains.py +0 -214
  14. futureagi-0.6.0/fi/evals/protect.py +0 -557
  15. futureagi-0.6.0/fi/evals/templates.py +0 -304
  16. futureagi-0.6.0/fi/evals/types.py +0 -116
  17. {futureagi-0.6.0 → futureagi-0.6.2}/LICENSE.md +0 -0
  18. {futureagi-0.6.0 → futureagi-0.6.2}/README.md +0 -0
  19. {futureagi-0.6.0 → futureagi-0.6.2}/fi/__init__.py +0 -0
  20. {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/__init__.py +0 -0
  21. {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/apikeys.py +0 -0
  22. {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/auth.py +0 -0
  23. {futureagi-0.6.0 → futureagi-0.6.2}/fi/api/types.py +0 -0
  24. {futureagi-0.6.0 → futureagi-0.6.2}/fi/client.py +0 -0
  25. {futureagi-0.6.0 → futureagi-0.6.2}/fi/datasets/__init__.py +0 -0
  26. {futureagi-0.6.0 → futureagi-0.6.2}/fi/datasets/types.py +0 -0
  27. {futureagi-0.6.0 → futureagi-0.6.2}/fi/kb/__init__.py +0 -0
  28. {futureagi-0.6.0 → futureagi-0.6.2}/fi/kb/client.py +0 -0
  29. {futureagi-0.6.0 → futureagi-0.6.2}/fi/kb/types.py +0 -0
  30. {futureagi-0.6.0 → futureagi-0.6.2}/fi/prompt/__init__.py +0 -0
  31. {futureagi-0.6.0 → futureagi-0.6.2}/fi/prompt/client.py +0 -0
  32. {futureagi-0.6.0 → futureagi-0.6.2}/fi/prompt/types.py +0 -0
  33. {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/__init__.py +0 -0
  34. {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/conversational_test_case.py +0 -0
  35. {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/general.py +0 -0
  36. {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/llm_test_case.py +0 -0
  37. {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/mllm_test_case.py +0 -0
  38. {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/unified_test_case.py +0 -0
  39. {futureagi-0.6.0 → futureagi-0.6.2}/fi/testcases/utils.py +0 -0
  40. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/__init__.py +0 -0
  41. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/constants.py +0 -0
  42. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/errors.py +0 -0
  43. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/executor.py +0 -0
  44. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/logging.py +0 -0
  45. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/routes.py +0 -0
  46. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/types.py +0 -0
  47. {futureagi-0.6.0 → futureagi-0.6.2}/fi/utils/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: futureagi
3
- Version: 0.6.0
3
+ Version: 0.6.2
4
4
  Summary: We help GenAI teams maintain high-accuracy for their Models in production.
5
5
  Author: Future AGI
6
6
  Author-email: no-reply@futureagi.com
@@ -19,8 +19,12 @@ from fi.datasets.types import (
19
19
  HuggingfaceDatasetConfig,
20
20
  Row,
21
21
  )
22
- from fi.evals.evaluator import EvalInfoResponseHandler
23
- from fi.evals.templates import EvalTemplate
22
+ try :
23
+ from fi.evals.evaluator import EvalInfoResponseHandler
24
+ from fi.evals.templates import EvalTemplate
25
+ except ImportError:
26
+ logging.warning(f"ai-evals is not installed. Please install it to add evaluations to your dataset.")
27
+
24
28
  from fi.utils.constants import (
25
29
  DATASET_TEMP_FILE_PREFIX,
26
30
  DATASET_TEMP_FILE_SUFFIX,
@@ -459,6 +463,11 @@ class Dataset(APIKeyAuth):
459
463
  reason_column: bool = False,
460
464
  config: Optional[Dict[str, Any]] = None,
461
465
  ) -> "Dataset":
466
+ try :
467
+ from fi.evals.evaluator import EvalInfoResponseHandler
468
+ from fi.evals.templates import EvalTemplate
469
+ except ImportError:
470
+ raise DatasetError("ai-evals is not installed. Please install it to add evaluations to your dataset.")
462
471
 
463
472
  if not self.dataset_config or not self.dataset_config.id:
464
473
  raise DatasetError("Dataset must be configured with an ID to add an evaluation.")
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "futureagi"
3
- version = "0.6.0"
3
+ version = "0.6.2"
4
4
  description = "We help GenAI teams maintain high-accuracy for their Models in production."
5
5
  authors = ["Future AGI <no-reply@futureagi.com>"]
6
6
  readme = "README.md"
@@ -8,7 +8,8 @@ packages = [
8
8
  { include = "fi" }
9
9
  ]
10
10
  exclude = [
11
- "fi/integrations/otel"
11
+ "fi/integrations/otel",
12
+ "fi/evals"
12
13
  ]
13
14
 
14
15
  [tool.poetry.dependencies]
@@ -1,20 +0,0 @@
1
- import inspect
2
-
3
- from fi.evals.evaluator import Evaluator, evaluate, list_evaluations # noqa: F401
4
- from fi.evals.protect import Protect, protect # noqa: F401
5
- from fi.evals.templates import * # noqa: F403, F401
6
- from fi.evals.metrics import BLEUScore, ROUGEScore, NumericDiff, LevenshteinDistance, EmbeddingSimilarity, SemanticListContains, AggregatedMetric
7
-
8
- # Dynamically generate __all__ from imported templates
9
- _globals = globals()
10
- evaluation_template_names = [
11
- name
12
- for name, obj in _globals.items()
13
- if inspect.isclass(obj) and obj.__module__ == "fi.evals.templates"
14
- ]
15
-
16
- # Add the clients separately
17
- client_names = ["Evaluator", "Protect", "evaluate", "protect", "list_evaluations"]
18
-
19
- # Combine and sort for consistency
20
- __all__ = sorted(evaluation_template_names + client_names)
@@ -1,375 +0,0 @@
1
- import inspect
2
- import json
3
- import logging
4
- from functools import lru_cache
5
- from typing import Any, Dict, List, Optional, Union
6
- from concurrent.futures import ThreadPoolExecutor, as_completed, TimeoutError
7
- from collections import defaultdict
8
- import os
9
-
10
- from requests import Response
11
-
12
- from fi.api.auth import APIKeyAuth, ResponseHandler
13
- from fi.api.types import HttpMethod, RequestConfig
14
- from fi.evals.templates import EvalTemplate
15
- from fi.evals.types import BatchRunResult, EvalResult, EvalResultMetric
16
- from fi.testcases import TestCase, MLLMImage, MLLMAudio
17
- from fi.utils.errors import InvalidAuthError
18
- from fi.utils.routes import Routes
19
- from fi.utils.utils import get_keys_from_env, get_base_url_from_env
20
-
21
-
22
- class EvalResponseHandler(ResponseHandler[BatchRunResult, None]):
23
- """Handles responses for evaluation requests"""
24
-
25
- @classmethod
26
- def _parse_success(cls, response: Response) -> BatchRunResult:
27
- return cls.convert_to_batch_results(response.json())
28
-
29
- @classmethod
30
- def _handle_error(cls, response: Response) -> None:
31
- if response.status_code == 400:
32
- raise Exception(
33
- f"Evaluation failed with a 400 Bad Request. Please check your input data and evaluation configuration. Response: {response.text}"
34
- )
35
- elif response.status_code == 403:
36
- raise InvalidAuthError()
37
- else:
38
- raise Exception(
39
- f"Error in evaluation: {response.status_code}, response: {response.text}"
40
- )
41
-
42
- @classmethod
43
- def convert_to_batch_results(cls, response: Dict[str, Any]) -> BatchRunResult:
44
- """
45
- Convert API response to BatchRunResult
46
-
47
- Args:
48
- response: Raw API response dictionary
49
-
50
- Returns:
51
- BatchRunResult containing evaluation results
52
- """
53
- eval_results = []
54
-
55
- for result in response.get("result", {}):
56
- for evaluation in result.get("evaluations", []):
57
- new_metadata = {}
58
- if evaluation.get("metadata"):
59
- if isinstance(evaluation.get("metadata"), dict):
60
- metadata = evaluation.get("metadata")
61
- elif isinstance(evaluation.get("metadata"), str):
62
- metadata = json.loads(evaluation.get("metadata"))
63
- else:
64
- metadata = {}
65
- new_metadata["usage"] = metadata.get("usage", {})
66
- new_metadata["cost"] = metadata.get("cost", {})
67
- new_metadata["explanation"] = metadata.get("explanation", {})
68
- eval_results.append(
69
- EvalResult(
70
- data=evaluation.get("data"),
71
- failure=evaluation.get("failure"),
72
- reason=evaluation.get("reason", ""),
73
- runtime=evaluation.get("runtime", 0),
74
- metadata=new_metadata,
75
- metrics=[
76
- EvalResultMetric(id=metric["id"], value=metric["value"])
77
- for metric in evaluation.get("metrics", [])
78
- ],
79
- )
80
- )
81
-
82
- return BatchRunResult(eval_results=eval_results)
83
-
84
-
85
- class EvalInfoResponseHandler(ResponseHandler[dict, None]):
86
- """Handles responses for evaluation info requests"""
87
-
88
- @classmethod
89
- def _parse_success(cls, response: Response) -> dict:
90
- data = response.json()
91
- if "result" in data:
92
- return data["result"]
93
- else:
94
- raise Exception(f"Failed to get evaluation info: {data}")
95
-
96
- @classmethod
97
- def _handle_error(cls, response: Response) -> None:
98
- if response.status_code == 400:
99
- response.raise_for_status()
100
- if response.status_code == 403:
101
- raise InvalidAuthError()
102
- raise Exception(f"Failed to get evaluation info: {response.status_code}")
103
-
104
-
105
- class Evaluator(APIKeyAuth):
106
- """Client for evaluating LLM test cases"""
107
-
108
- def __init__(
109
- self,
110
- fi_api_key: Optional[str] = None,
111
- fi_secret_key: Optional[str] = None,
112
- fi_base_url: Optional[str] = None,
113
- **kwargs,
114
- ) -> None:
115
- """
116
- Initialize the Eval Client
117
-
118
- Args:
119
- fi_api_key: API key
120
- fi_secret_key: Secret key
121
- fi_base_url: Base URL
122
-
123
- Keyword Args:
124
- timeout: Optional timeout value in seconds (default: 200)
125
- max_queue_bound: Optional maximum queue size (default: 5000)
126
- max_workers: Optional maximum number of workers (default: 8)
127
- """
128
- super().__init__(fi_api_key, fi_secret_key, fi_base_url, **kwargs)
129
- self._max_workers = kwargs.get("max_workers", 8) # Default to 8 if not provided
130
-
131
- def evaluate(
132
- self,
133
- eval_templates: Union[str, type[EvalTemplate]],
134
- inputs: Union[
135
- TestCase,
136
- List[TestCase],
137
- Dict[str, Any],
138
- List[Dict[str, Any]],
139
- ],
140
- timeout: Optional[int] = None,
141
- model_name: Optional[str] = None,
142
- ) -> BatchRunResult:
143
- """
144
- Run a single or batch of evaluations independently
145
-
146
- Args:
147
- eval_templates: Evaluation name string (e.g., "Factual Accuracy")
148
- inputs: Single test case or list of test cases
149
- timeout: Optional timeout value for the evaluation
150
- model_name: Optional model name to use for the evaluation for Future AGI Agents
151
-
152
- Returns:
153
- BatchRunResult containing evaluation results
154
-
155
- Raises:
156
- ValidationError: If the inputs do not match the evaluation templates
157
- Exception: If the API request fails
158
- """
159
- if not isinstance(inputs, list):
160
- inputs = [inputs]
161
-
162
- def _extract_name(t) -> str | None:
163
- if isinstance(t, str):
164
- return t
165
- if isinstance(t, EvalTemplate):
166
- return t.eval_name
167
- if inspect.isclass(t) and issubclass(t, EvalTemplate):
168
- return t.eval_name
169
- return None
170
- eval_name = _extract_name(
171
- eval_templates[0] if isinstance(eval_templates, list) else eval_templates
172
- )
173
- if eval_name is None:
174
- raise TypeError(
175
- "Unsupported eval_templates argument. "
176
- "Expect eval template class/obj or name str."
177
- )
178
-
179
-
180
-
181
- transformed_api_inputs = defaultdict(list)
182
-
183
- if not inputs:
184
- # Allows an empty transformed_api_inputs if the original inputs list was empty.
185
- pass
186
- else:
187
- for tc in inputs:
188
- dumped_tc: Dict[str, Any]
189
- # Handle both TestCase objects and raw dictionaries
190
- if isinstance(tc, TestCase):
191
- # TestCase object
192
- dumped_tc = tc.model_dump(exclude_none=True, by_alias=False)
193
- elif isinstance(tc, dict):
194
- # Raw dictionary
195
- processed_tc = {}
196
- for key, value in tc.items():
197
- if isinstance(value, str):
198
- original_value = value # Keep a copy for fallback
199
- try:
200
- mllm_image = MLLMImage(url=value)
201
- processed_tc[key] = mllm_image.url
202
- logging.debug(f"Processed key '{key}' as MLLMImage.")
203
- continue
204
- except ValueError as e_img:
205
- logging.debug(f"Key '{key}' not processed as MLLMImage (Error: {e_img}). Trying MLLMAudio.")
206
- try:
207
- mllm_audio = MLLMAudio(url=value)
208
- if not mllm_audio.is_plain_text:
209
- processed_tc[key] = mllm_audio.url
210
- logging.debug(f"Processed key '{key}' as MLLMAudio.")
211
- continue
212
- else:
213
- logging.debug(f"Key '{key}' treated as plain text by MLLMAudio. Using original value.")
214
- processed_tc[key] = original_value
215
- except ValueError as e_audio:
216
- logging.debug(f"Key '{key}' not processed as MLLMAudio (Error: {e_audio}). Using original value.")
217
- processed_tc[key] = original_value
218
- except Exception as e_gen_audio:
219
- logging.warning(f"Unexpected error processing key '{key}' as MLLMAudio (Error: {e_gen_audio}). Using original value.")
220
- processed_tc[key] = original_value
221
- except Exception as e_gen_image:
222
- logging.warning(f"Unexpected error processing key '{key}' as MLLMImage (Error: {e_gen_image}). Using original value.")
223
- processed_tc[key] = original_value
224
- else:
225
- # Not a string, keep original value
226
- processed_tc[key] = value
227
- dumped_tc = processed_tc
228
- else:
229
- raise TypeError(
230
- f"Invalid input type: {type(tc)}. Each input must be a TestCase object or a dictionary with valid keys."
231
- )
232
-
233
- for key, value in dumped_tc.items():
234
- transformed_api_inputs[key].append(value)
235
-
236
- final_api_payload = {
237
- "eval_name": eval_name,
238
- "inputs": dict(transformed_api_inputs),
239
- "model": model_name,
240
- }
241
-
242
-
243
- all_results = []
244
- failed_inputs = []
245
- with ThreadPoolExecutor(max_workers=self._max_workers) as executor:
246
- # Submit the batch only once
247
- future = executor.submit(
248
- self.request,
249
- config=RequestConfig(
250
- method=HttpMethod.POST,
251
- url=f"{self._base_url}/{Routes.evaluatev2.value}",
252
- json=final_api_payload,
253
- timeout=timeout or self._default_timeout,
254
- ),
255
- response_handler=EvalResponseHandler,
256
- )
257
- future_to_input = {future: inputs} # map single future to all inputs
258
-
259
- for future in as_completed(future_to_input):
260
- try:
261
- response: BatchRunResult = future.result(timeout=timeout or self._default_timeout)
262
- all_results.extend(response.eval_results)
263
- except TimeoutError:
264
- input_case = future_to_input[future]
265
- logging.error(f"Evaluation timed out for input: {input_case}")
266
- failed_inputs.append(input_case)
267
-
268
- except Exception as exc:
269
- input_case = future_to_input[future]
270
- logging.error(f"Evaluation failed for input {input_case}: {str(exc)}")
271
- failed_inputs.append(input_case)
272
-
273
- if failed_inputs:
274
- logging.warning(f"Failed to evaluate {len(failed_inputs)} inputs out of {len(inputs)} total inputs")
275
-
276
- return BatchRunResult(eval_results=all_results)
277
-
278
- def _validate_inputs(
279
- self,
280
- inputs: List[TestCase],
281
- eval_objects: List[EvalTemplate],
282
- ):
283
- """
284
- Validate the inputs against the evaluation templates
285
-
286
- Args:
287
- inputs: List of test cases to validate
288
- eval_objects: List of evaluation templates to validate against
289
-
290
- Returns:
291
- bool: True if validation passes
292
-
293
- Raises:
294
- Exception: If validation fails or templates don't share common tags
295
- """
296
-
297
- # First validate that all eval objects share at least one common tag
298
- if len(eval_objects) > 1:
299
- # Get sets of tags for each eval object
300
- tag_sets = [set(obj.eval_tags) for obj in eval_objects]
301
-
302
- # Find intersection of all tag sets
303
- common_tags = set.intersection(*tag_sets)
304
-
305
- if not common_tags:
306
- template_names = [obj.name for obj in eval_objects]
307
- raise Exception(
308
- f"Evaluation templates {template_names} must share at least one common tag. "
309
- f"Current tags for each template: {[list(tags) for tags in tag_sets]}"
310
- )
311
-
312
- # Then validate each eval object's required inputs
313
- for eval_object in eval_objects:
314
- eval_object.validate_input(inputs)
315
-
316
- return True
317
-
318
- def _get_eval_configs(
319
- self, eval_templates: Union[str, List[str]]
320
- ) -> List[EvalTemplate]:
321
- if isinstance(eval_templates, str):
322
- eval_templates = [eval_templates]
323
-
324
- for template in eval_templates:
325
- eval_info = self._get_eval_info(template)
326
- template.eval_id = eval_info["eval_id"]
327
- template.name = eval_info["name"]
328
- template.description = eval_info["description"]
329
- template.eval_tags = eval_info["eval_tags"]
330
- template.required_keys = eval_info["config"]["required_keys"]
331
- template.output = eval_info["config"]["output"]
332
- template.eval_type_id = eval_info["config"]["eval_type_id"]
333
- template.config_schema = (
334
- eval_info["config"]["config"] if "config" in eval_info["config"] else {}
335
- )
336
- template.criteria = eval_info["criteria"]
337
- template.choices = eval_info["choices"]
338
- template.multi_choice = eval_info["multi_choice"]
339
- return eval_templates
340
-
341
- @lru_cache(maxsize=100)
342
- def _get_eval_info(self, eval_name: str) -> Dict[str, Any]:
343
- url = (
344
- self._base_url
345
- + "/"
346
- + Routes.get_eval_templates.value
347
- )
348
- response = self.request(
349
- config=RequestConfig(method=HttpMethod.GET, url=url),
350
- response_handler=EvalInfoResponseHandler,
351
- )
352
- eval_info = next((item for item in response if item["name"] == eval_name), None)
353
- if eval_info is None:
354
- raise KeyError(f"Evaluation template '{eval_name}' not found in registry")
355
- if not eval_info:
356
- raise Exception(f"Evaluation template with name '{eval_name}' not found")
357
- return eval_info
358
-
359
- def list_evaluations(self):
360
- """
361
- Fetch information about all available evaluation templates by getting eval_info
362
- for each template class defined in templates.py.
363
-
364
- Returns:
365
- List[Dict[str, Any]]: List of evaluation template information dictionaries
366
- """
367
- config = RequestConfig(method=HttpMethod.GET,
368
- url=f"{self._base_url}/{Routes.get_eval_templates.value}")
369
-
370
- response = self.request(config=config, response_handler=EvalInfoResponseHandler)
371
-
372
- return response
373
- evaluate = lambda eval_templates, inputs, timeout=None: Evaluator().evaluate(eval_templates, inputs, timeout)
374
-
375
- list_evaluations = lambda: Evaluator().list_evaluations()
@@ -1,17 +0,0 @@
1
- from fi.evals.metrics.bleu import BLEUScore
2
- from fi.evals.metrics.rouge import ROUGEScore
3
- from fi.evals.metrics.numeric_diff import NumericDiff
4
- from fi.evals.metrics.laveshtein import LevenshteinDistance
5
- from fi.evals.metrics.embedding_similarity import EmbeddingSimilarity
6
- from fi.evals.metrics.semantic_list_contains import SemanticListContains
7
- from fi.evals.metrics.aggregated_metric import AggregatedMetric
8
-
9
- __all__ = [
10
- "BLEUScore",
11
- "ROUGEScore",
12
- "NumericDiff",
13
- "LevenshteinDistance",
14
- "EmbeddingSimilarity",
15
- "SemanticListContains",
16
- "AggregatedMetric"
17
- ]
@@ -1,178 +0,0 @@
1
- import string
2
- from typing import List, Union, Any, Dict, Optional, Tuple
3
- import json
4
-
5
- from fi.evals.templates import EvalTemplate
6
- from fi.evals.types import EvalResult, EvalResultMetric, BatchRunResult
7
- from fi.testcases import TestCase
8
-
9
-
10
- class AggregatedMetric(EvalTemplate):
11
- """
12
- AggregatedMetric combines multiple metric evaluators into a single aggregated score.
13
-
14
- Possible values for aggregator:
15
- - 'average': Computes the mean of all metric scores.
16
- - 'weighted_average': Computes the weighted mean using provided weights.
17
-
18
- Example config:
19
- config = {
20
- "aggregator": "average", # or "weighted_average"
21
- "weights": [0.5, 0.5], # Only for weighted_average
22
- "metrics": [metric1, metric2]
23
- }
24
-
25
- Example usage:
26
- agg_metric = AggregatedMetric(config)
27
- agg_metric.validate_input(test_cases)
28
- result = agg_metric.evaluate(test_cases)
29
- """
30
- EVAL_ID = "82"
31
- NAME = "Aggregated Metric"
32
- DESCRIPTION = "Combines multiple metrics into a single aggregated score"
33
- EVAL_TAGS = ["metric", "aggregation", "composite", "multi-metric"]
34
- REQUIRED_KEYS = ["response", "expected_text"]
35
- OUTPUT = "score"
36
- EVAL_TYPE_ID = "metric"
37
- DEFAULT_CONFIG_SCHEMA = {
38
- "aggregator": "average",
39
- "weights": None,
40
- "metrics": []
41
- }
42
- CRITERIA = "Aggregates multiple evaluation metrics into a combined score"
43
- CHOICES = []
44
- MULTI_CHOICE = False
45
- SUPPORTED_AGGREGATORS = ["average", "weighted_average"]
46
-
47
- def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
48
- config = config or {}
49
- self.aggregator = config.get("aggregator", self.DEFAULT_CONFIG_SCHEMA["aggregator"])
50
- self.weights = config.get("weights", self.DEFAULT_CONFIG_SCHEMA["weights"])
51
- self.metrics = config.get("metrics", self.DEFAULT_CONFIG_SCHEMA["metrics"])
52
-
53
- # Set required instance attributes for EvalTemplate
54
- self.name = self.NAME
55
- self.description = self.DESCRIPTION
56
- self.eval_id = self.EVAL_ID
57
- self.eval_tags = self.EVAL_TAGS
58
- self.required_keys = self.REQUIRED_KEYS
59
- self.output = self.OUTPUT
60
- self.eval_type_id = self.EVAL_TYPE_ID
61
- self.config_schema = self.DEFAULT_CONFIG_SCHEMA
62
- self.criteria = self.CRITERIA
63
- self.choices = self.CHOICES
64
- self.multi_choice = self.MULTI_CHOICE
65
-
66
- if not self.metrics:
67
- raise ValueError("AggregatedMetric requires at least one metric evaluator")
68
-
69
- if self.aggregator not in self.SUPPORTED_AGGREGATORS:
70
- raise ValueError(f"Unsupported aggregator: {self.aggregator}")
71
- if self.aggregator == "weighted_average":
72
- if self.weights is None or len(self.weights) != len(self.metrics):
73
- raise ValueError("Weights required for weighted_average and must match the number of metrics")
74
-
75
- self.metric_names = [getattr(m, 'NAME', getattr(m, 'name', str(m))) for m in self.metrics]
76
-
77
- super().__init__(config)
78
-
79
- def validate_input(self, inputs: List[TestCase]):
80
- for test_case in inputs:
81
- if not hasattr(test_case, "response") or test_case.response is None:
82
- raise ValueError("TestCase must have a 'response' field for AggregatedMetric evaluation")
83
- if not hasattr(test_case, "expected_text") or test_case.expected_text is None:
84
- raise ValueError("TestCase must have an 'expected_text' field for AggregatedMetric evaluation")
85
- for metric in self.metrics:
86
- if hasattr(metric, 'validate_input'):
87
- metric.validate_input(inputs)
88
- return True
89
-
90
- def evaluate(self, inputs: List[TestCase]) -> BatchRunResult:
91
- try:
92
- self.validate_input(inputs)
93
- except Exception as e:
94
- eval_result = EvalResult(
95
- data=[0.0, {}],
96
- failure=True,
97
- reason=f"Input validation failed: {str(e)}",
98
- runtime=0,
99
- metadata={
100
- "aggregator": self.aggregator,
101
- "metrics": self.metric_names,
102
- "weights": self.weights,
103
- "individual_scores": {}
104
- },
105
- metrics=[
106
- EvalResultMetric(
107
- id="aggregated_metric_score",
108
- value=0.0
109
- )
110
- ]
111
- )
112
- return BatchRunResult(eval_results=[eval_result])
113
-
114
- eval_results = []
115
- for test_case in inputs:
116
- try:
117
- metric_results = []
118
- for metric in self.metrics:
119
- try:
120
- if hasattr(metric, 'evaluate'):
121
- result = metric.evaluate([test_case])
122
- if result.eval_results and result.eval_results[0].metrics:
123
- metric_value = result.eval_results[0].metrics[0].value
124
- metric_results.append(self._normalize_score(metric_value))
125
- else:
126
- metric_results.append(0.0)
127
- else:
128
- metric_results.append(0.0)
129
- except Exception as metric_exc:
130
- metric_results.append(0.0)
131
- if self.aggregator == "average":
132
- aggregated_score = sum(metric_results) / len(metric_results) if metric_results else 0.0
133
- elif self.aggregator == "weighted_average":
134
- weighted_sum = sum(w * s for w, s in zip(self.weights, metric_results))
135
- total_weight = sum(self.weights)
136
- aggregated_score = weighted_sum / total_weight if total_weight > 0 else 0.0
137
- else:
138
- aggregated_score = 0.0
139
- metric_details = {
140
- name: score for name, score in zip(self.metric_names, metric_results)
141
- }
142
- failure = False
143
- reason = ""
144
- except Exception as e:
145
- aggregated_score = 0.0
146
- metric_details = {}
147
- failure = True
148
- reason = f"Evaluation failed: {str(e)}"
149
- eval_result = EvalResult(
150
- data=[aggregated_score, metric_details],
151
- failure=failure,
152
- reason=reason,
153
- runtime=0,
154
- metadata={
155
- "aggregator": self.aggregator,
156
- "metrics": self.metric_names,
157
- "weights": self.weights,
158
- "individual_scores": metric_details
159
- },
160
- metrics=[
161
- EvalResultMetric(
162
- id="aggregated_metric_score",
163
- value=aggregated_score
164
- )
165
- ]
166
- )
167
- eval_results.append(eval_result)
168
- return BatchRunResult(eval_results=eval_results)
169
-
170
- def _normalize_score(self, value: Any) -> float:
171
- """Convert various score types to a float between 0 and 1"""
172
- if isinstance(value, bool):
173
- return 1.0 if value else 0.0
174
- try:
175
- float_value = float(value)
176
- return max(0.0, min(1.0, float_value))
177
- except (ValueError, TypeError):
178
- return 0.0