futureagi 0.4.2__tar.gz → 0.4.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. {futureagi-0.4.2 → futureagi-0.4.3}/PKG-INFO +1 -1
  2. {futureagi-0.4.2 → futureagi-0.4.3}/fi/datasets/client.py +62 -69
  3. {futureagi-0.4.2 → futureagi-0.4.3}/fi/evals/__init__.py +2 -0
  4. futureagi-0.4.3/fi/evals/protect.py +403 -0
  5. {futureagi-0.4.2 → futureagi-0.4.3}/fi/evals/templates.py +16 -0
  6. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/routes.py +1 -1
  7. {futureagi-0.4.2 → futureagi-0.4.3}/pyproject.toml +1 -1
  8. {futureagi-0.4.2 → futureagi-0.4.3}/LICENSE.md +0 -0
  9. {futureagi-0.4.2 → futureagi-0.4.3}/README.md +0 -0
  10. {futureagi-0.4.2 → futureagi-0.4.3}/fi/__init__.py +0 -0
  11. {futureagi-0.4.2 → futureagi-0.4.3}/fi/api/__init__.py +0 -0
  12. {futureagi-0.4.2 → futureagi-0.4.3}/fi/api/apikeys.py +0 -0
  13. {futureagi-0.4.2 → futureagi-0.4.3}/fi/api/auth.py +0 -0
  14. {futureagi-0.4.2 → futureagi-0.4.3}/fi/api/types.py +0 -0
  15. {futureagi-0.4.2 → futureagi-0.4.3}/fi/client.py +0 -0
  16. {futureagi-0.4.2 → futureagi-0.4.3}/fi/datasets/__init__.py +0 -0
  17. {futureagi-0.4.2 → futureagi-0.4.3}/fi/datasets/types.py +0 -0
  18. {futureagi-0.4.2 → futureagi-0.4.3}/fi/evals/evaluator.py +0 -0
  19. {futureagi-0.4.2 → futureagi-0.4.3}/fi/evals/types.py +0 -0
  20. {futureagi-0.4.2 → futureagi-0.4.3}/fi/integrations/__init__.py +0 -0
  21. {futureagi-0.4.2 → futureagi-0.4.3}/fi/integrations/providers/__init__.py +0 -0
  22. {futureagi-0.4.2 → futureagi-0.4.3}/fi/integrations/providers/types.py +0 -0
  23. {futureagi-0.4.2 → futureagi-0.4.3}/fi/prompt/__init__.py +0 -0
  24. {futureagi-0.4.2 → futureagi-0.4.3}/fi/prompt/client.py +0 -0
  25. {futureagi-0.4.2 → futureagi-0.4.3}/fi/prompt/types.py +0 -0
  26. {futureagi-0.4.2 → futureagi-0.4.3}/fi/testcases/__init__.py +0 -0
  27. {futureagi-0.4.2 → futureagi-0.4.3}/fi/testcases/conversational_test_case.py +0 -0
  28. {futureagi-0.4.2 → futureagi-0.4.3}/fi/testcases/general.py +0 -0
  29. {futureagi-0.4.2 → futureagi-0.4.3}/fi/testcases/llm_test_case.py +0 -0
  30. {futureagi-0.4.2 → futureagi-0.4.3}/fi/testcases/mllm_test_case.py +0 -0
  31. {futureagi-0.4.2 → futureagi-0.4.3}/fi/testcases/utils.py +0 -0
  32. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/__init__.py +0 -0
  33. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/constants.py +0 -0
  34. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/errors.py +0 -0
  35. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/executor.py +0 -0
  36. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/logging.py +0 -0
  37. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/types.py +0 -0
  38. {futureagi-0.4.2 → futureagi-0.4.3}/fi/utils/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: futureagi
3
- Version: 0.4.2
3
+ Version: 0.4.3
4
4
  Summary: We help GenAI teams maintain high-accuracy for their Models in production.
5
5
  Author: Future AGI
6
6
  Author-email: no-reply@futureagi.com
@@ -1,6 +1,6 @@
1
1
  import os
2
2
  import re
3
- from typing import List, Optional, Union, Dict, Any
3
+ from typing import Any, Dict, List, Optional, Union
4
4
 
5
5
  import pandas as pd
6
6
  from requests import Response
@@ -16,6 +16,8 @@ from fi.datasets.types import (
16
16
  HuggingfaceDatasetConfig,
17
17
  Row,
18
18
  )
19
+ from fi.evals.evaluator import EvalInfoResponseHandler
20
+ from fi.evals.templates import EvalTemplate
19
21
  from fi.utils.constants import (
20
22
  DATASET_TEMP_FILE_PREFIX,
21
23
  DATASET_TEMP_FILE_SUFFIX,
@@ -24,8 +26,6 @@ from fi.utils.constants import (
24
26
  from fi.utils.errors import InvalidAuthError, NoDatasetFound
25
27
  from fi.utils.routes import Routes
26
28
  from fi.utils.utils import get_tempfile_path
27
- from fi.evals.templates import EvalTemplate
28
- from fi.evals.evaluator import EvalInfoResponseHandler
29
29
 
30
30
 
31
31
  class DatasetResponseHandler(ResponseHandler[DatasetConfig, DatasetTable]):
@@ -134,7 +134,7 @@ class DatasetClient(APIKeyAuth):
134
134
  """
135
135
 
136
136
  _dataset_id_cache = {}
137
- __static_attributes__ = ['get_dataset_config']
137
+ __static_attributes__ = ["get_dataset_config"]
138
138
 
139
139
  def __init__(
140
140
  self,
@@ -230,29 +230,26 @@ class DatasetClient(APIKeyAuth):
230
230
 
231
231
  self._add_rows(rows)
232
232
  return self
233
-
234
- def get_column_id(self, column_name: str)-> Optional[str]:
233
+
234
+ def get_column_id(self, column_name: str) -> Optional[str]:
235
235
  if not self.dataset_config or not self.dataset_config.id:
236
236
  raise ValueError("No dataset configured")
237
-
237
+
238
238
  url = f"{self._base_url}/{Routes.dataset_table.value.format(dataset_id=str(self.dataset_config.id))}"
239
239
  dataset_table = self.request(
240
240
  config=RequestConfig(
241
- method=HttpMethod.POST,
241
+ method=HttpMethod.POST,
242
242
  url=url,
243
- json={
244
- "page_size": 1,
245
- "current_page_index": 0
246
- }
243
+ json={"page_size": 1, "current_page_index": 0},
247
244
  ),
248
- response_handler=DatasetResponseHandler
245
+ response_handler=DatasetResponseHandler,
249
246
  )
250
-
247
+
251
248
  for column in dataset_table.columns:
252
249
  if column.name == column_name:
253
250
  return str(column.id)
254
251
  return None
255
-
252
+
256
253
  def add_run_prompt(
257
254
  self,
258
255
  name: str,
@@ -267,7 +264,7 @@ class DatasetClient(APIKeyAuth):
267
264
  top_p: float = 1,
268
265
  tools: Optional[List[Dict]] = None,
269
266
  tool_choice: Optional[Any] = None,
270
- response_format: Optional[Dict] = None
267
+ response_format: Optional[Dict] = None,
271
268
  ) -> "DatasetClient":
272
269
 
273
270
  if not self.dataset_config:
@@ -275,23 +272,27 @@ class DatasetClient(APIKeyAuth):
275
272
 
276
273
  processed_messages = []
277
274
  referenced_columns = set()
278
-
275
+
279
276
  for msg in messages:
280
277
  if "role" not in msg:
281
278
  msg["role"] = "user"
282
-
279
+
283
280
  if "content" in msg:
284
281
  content = msg["content"]
285
- column_refs = re.findall(r'\{\{(.*?)\}\}', content)
286
-
282
+ column_refs = re.findall(r"\{\{(.*?)\}\}", content)
283
+
287
284
  for col_name in column_refs:
288
285
  col_id = self.get_column_id(col_name)
289
286
  if not col_id:
290
- raise ValueError(f"Referenced column '{col_name}' not found in dataset")
287
+ raise ValueError(
288
+ f"Referenced column '{col_name}' not found in dataset"
289
+ )
291
290
  referenced_columns.add(col_name)
292
- content = content.replace(f"{{{{{col_name}}}}}", f"{{{{{col_id}}}}}")
291
+ content = content.replace(
292
+ f"{{{{{col_name}}}}}", f"{{{{{col_id}}}}}"
293
+ )
293
294
  msg["content"] = content
294
-
295
+
295
296
  processed_messages.append(msg)
296
297
 
297
298
  config = {
@@ -307,7 +308,7 @@ class DatasetClient(APIKeyAuth):
307
308
  "presencePenalty": presence_penalty,
308
309
  "frequencyPenalty": frequency_penalty,
309
310
  "topP": top_p,
310
- }
311
+ },
311
312
  }
312
313
 
313
314
  if tools:
@@ -320,9 +321,9 @@ class DatasetClient(APIKeyAuth):
320
321
  url = f"{self._base_url}/{Routes.dataset_add_run_prompt_column.value}"
321
322
  self.request(
322
323
  config=RequestConfig(method=HttpMethod.POST, url=url, json=config),
323
- response_handler=DatasetResponseHandler
324
+ response_handler=DatasetResponseHandler,
324
325
  )
325
-
326
+
326
327
  return self
327
328
 
328
329
  def add_evaluation(
@@ -336,33 +337,27 @@ class DatasetClient(APIKeyAuth):
336
337
  save_as_template: bool = False,
337
338
  run: bool = True,
338
339
  reason_column: bool = False,
339
- config: Optional[Dict[str, Any]] = None
340
+ config: Optional[Dict[str, Any]] = None,
340
341
  ) -> "DatasetClient":
341
342
 
342
343
  if not self.dataset_config or not self.dataset_config.id:
343
344
  raise ValueError("No dataset configured")
344
345
 
345
- template_classes = {
346
- cls.__name__: cls
347
- for cls in EvalTemplate.__subclasses__()
348
- }
346
+ template_classes = {cls.__name__: cls for cls in EvalTemplate.__subclasses__()}
349
347
  if eval_template not in template_classes:
350
348
  raise ValueError(f"Unknown template name: {eval_template}")
351
-
349
+
352
350
  eval_id = template_classes[eval_template].eval_id
353
351
 
354
352
  url = f"{self._base_url}/sdk/api/v1/eval/{eval_id}/"
355
353
  template_details = self.request(
356
- config=RequestConfig(
357
- method=HttpMethod.GET,
358
- url=url
359
- ),
360
- response_handler=EvalInfoResponseHandler
354
+ config=RequestConfig(method=HttpMethod.GET, url=url),
355
+ response_handler=EvalInfoResponseHandler,
361
356
  )
362
-
357
+
363
358
  template_id = template_details["id"]
364
359
  required_keys = template_details["config"]["required_keys"]
365
-
360
+
366
361
  mapping = {}
367
362
 
368
363
  if "input" in required_keys:
@@ -372,7 +367,7 @@ class DatasetClient(APIKeyAuth):
372
367
  if not input_column_id:
373
368
  raise ValueError(f"Column '{input_column_name}' not found in dataset")
374
369
  mapping["input"] = input_column_id
375
-
370
+
376
371
  if "output" in required_keys or "response" in required_keys:
377
372
  if not output_column_name:
378
373
  raise ValueError(f"Output column required for {eval_template}")
@@ -383,7 +378,7 @@ class DatasetClient(APIKeyAuth):
383
378
  mapping["output"] = output_column_id
384
379
  else:
385
380
  mapping["response"] = output_column_id
386
-
381
+
387
382
  if "context" in required_keys:
388
383
  if not context_column_name:
389
384
  raise ValueError(f"Context column required for {eval_template}")
@@ -391,13 +386,17 @@ class DatasetClient(APIKeyAuth):
391
386
  if not context_column_id:
392
387
  raise ValueError(f"Column '{context_column_name}' not found in dataset")
393
388
  mapping["context"] = context_column_id
394
-
389
+
395
390
  if "expected_response" in required_keys:
396
391
  if not expected_column_name:
397
- raise ValueError(f"Expected response column required for {eval_template}")
392
+ raise ValueError(
393
+ f"Expected response column required for {eval_template}"
394
+ )
398
395
  expected_column_id = self.get_column_id(expected_column_name)
399
396
  if not expected_column_id:
400
- raise ValueError(f"Column '{expected_column_name}' not found in dataset")
397
+ raise ValueError(
398
+ f"Column '{expected_column_name}' not found in dataset"
399
+ )
401
400
  mapping["expected_response"] = expected_column_id
402
401
 
403
402
  eval_config = {
@@ -408,20 +407,16 @@ class DatasetClient(APIKeyAuth):
408
407
  "config": {
409
408
  "mapping": mapping,
410
409
  "config": config or {},
411
- "reasonColumn": reason_column
412
- }
410
+ "reasonColumn": reason_column,
411
+ },
413
412
  }
414
413
 
415
414
  url = f"{self._base_url}/model-hub/develops/{str(self.dataset_config.id)}/add_user_eval/"
416
415
  self.request(
417
- config=RequestConfig(
418
- method=HttpMethod.POST,
419
- url=url,
420
- json=eval_config
421
- ),
422
- response_handler=DatasetResponseHandler
416
+ config=RequestConfig(method=HttpMethod.POST, url=url, json=eval_config),
417
+ response_handler=DatasetResponseHandler,
423
418
  )
424
-
419
+
425
420
  return self
426
421
 
427
422
  def add_optimization(
@@ -429,7 +424,7 @@ class DatasetClient(APIKeyAuth):
429
424
  optimization_name: str,
430
425
  prompt_column_name: str,
431
426
  optimize_type: str = "PROMPT_TEMPLATE",
432
- model_config: Optional[Dict[str, Any]] = None
427
+ model_config: Optional[Dict[str, Any]] = None,
433
428
  ) -> "DatasetClient":
434
429
 
435
430
  if not self.dataset_config or not self.dataset_config.id:
@@ -437,17 +432,17 @@ class DatasetClient(APIKeyAuth):
437
432
 
438
433
  column_id = self.get_column_id(prompt_column_name)
439
434
  if not column_id:
440
- raise ValueError(f"Prompt Column '{prompt_column_name}' not found in dataset")
435
+ raise ValueError(
436
+ f"Prompt Column '{prompt_column_name}' not found in dataset"
437
+ )
441
438
 
442
- #requires mapping input for userEvalTemplateIds
439
+ # requires mapping input for userEvalTemplateIds
443
440
  metrics_url = f"{self._base_url}/model-hub/metrics/by-column/"
444
441
  metrics_response = self.request(
445
442
  config=RequestConfig(
446
- method=HttpMethod.GET,
447
- url=metrics_url,
448
- params={"column_id": column_id}
443
+ method=HttpMethod.GET, url=metrics_url, params={"column_id": column_id}
449
444
  ),
450
- response_handler=DatasetResponseHandler
445
+ response_handler=DatasetResponseHandler,
451
446
  )
452
447
 
453
448
  eval_template_ids = []
@@ -456,7 +451,9 @@ class DatasetClient(APIKeyAuth):
456
451
  eval_template_ids.append(metric["id"])
457
452
 
458
453
  if not eval_template_ids:
459
- raise ValueError(f"No evaluation templates found for column '{prompt_column_name}'")
454
+ raise ValueError(
455
+ f"No evaluation templates found for column '{prompt_column_name}'"
456
+ )
460
457
 
461
458
  payload = {
462
459
  "name": optimization_name,
@@ -464,19 +461,15 @@ class DatasetClient(APIKeyAuth):
464
461
  "optimizeType": optimize_type,
465
462
  "userEvalTemplateIds": eval_template_ids,
466
463
  "datasetId": str(self.dataset_config.id),
467
- "modelConfig": model_config or {}
464
+ "modelConfig": model_config or {},
468
465
  }
469
466
 
470
467
  opt_url = f"{self._base_url}/model-hub/optimisation/create/"
471
468
  self.request(
472
- config=RequestConfig(
473
- method=HttpMethod.POST,
474
- url=opt_url,
475
- json=payload
476
- ),
477
- response_handler=DatasetResponseHandler
469
+ config=RequestConfig(method=HttpMethod.POST, url=opt_url, json=payload),
470
+ response_handler=DatasetResponseHandler,
478
471
  )
479
-
472
+
480
473
  return self
481
474
 
482
475
  # Protected internal methods
@@ -1,4 +1,5 @@
1
1
  from fi.evals.evaluator import EvalClient # noqa: F401
2
+ from fi.evals.protect import ProtectClient # noqa: F401
2
3
  from fi.evals.templates import (
3
4
  PII,
4
5
  AgentJudge,
@@ -130,4 +131,5 @@ __all__ = [
130
131
  "Toxicity",
131
132
  "TranslationAccuracy",
132
133
  "EvalClient",
134
+ "ProtectClient",
133
135
  ]
@@ -0,0 +1,403 @@
1
+ import time
2
+ from concurrent.futures import ThreadPoolExecutor, TimeoutError, as_completed
3
+ from typing import Dict, List, Optional, Tuple
4
+
5
+ from fi.evals import EvalClient
6
+ from fi.evals.templates import (
7
+ DataPrivacyCompliance,
8
+ PromptInjection,
9
+ Sexist,
10
+ Tone,
11
+ Toxicity,
12
+ )
13
+ from fi.testcases import TestCase
14
+
15
+
16
+ class ProtectClient:
17
+ """Client for protecting against unwanted content using various metrics"""
18
+
19
+ def __init__(self, evaluator: EvalClient):
20
+ """
21
+ Initialize ProtectClient
22
+
23
+ Args:
24
+ evaluator: Instance of EvalClient to use for evaluations
25
+ """
26
+ self.evaluator = evaluator
27
+
28
+ # Map metric names to their corresponding template classes
29
+ self.metric_map = {
30
+ "Toxicity": Toxicity,
31
+ "Tone": Tone,
32
+ "Sexism": Sexist,
33
+ "Prompt Injection": PromptInjection,
34
+ "Data Privacy": DataPrivacyCompliance,
35
+ }
36
+ self.executor = ThreadPoolExecutor(max_workers=5)
37
+
38
+ def _check_rule_sync(
39
+ self, rule: Dict, test_case: TestCase
40
+ ) -> Tuple[str, bool, Optional[str]]:
41
+ """
42
+ Synchronous version of rule checking
43
+
44
+ Returns:
45
+ Tuple[str, bool, Optional[str]]: (rule_name, triggered, message if triggered)
46
+ """
47
+ # thread_name = threading.current_thread().name
48
+ # start_time = time.time()
49
+ # print(f"Starting rule check for {rule['metric']} in thread {thread_name} at {start_time}")
50
+
51
+ template_class = self.metric_map[rule["metric"]]
52
+ if rule["metric"] == "Data Privacy":
53
+ template = template_class(
54
+ config={"call_type": "protect", "check_internet": False}
55
+ )
56
+ # template = template_class(config={"check_internet": False})
57
+ else:
58
+ template = template_class(config={"call_type": "protect"})
59
+ # template = template_class(config={})
60
+
61
+ eval_result = self.evaluator.evaluate(template, test_case)
62
+
63
+ # end_time = time.time()
64
+ # print(f"Completed rule check for {rule['metric']} in thread {thread_name} at {end_time} (took {end_time - start_time:.2f}s)")
65
+
66
+ reason = ""
67
+
68
+ if eval_result.eval_results:
69
+ result = eval_result.eval_results[0]
70
+ detected_values = result.data
71
+
72
+ should_trigger = False
73
+ if rule["type"] == "any":
74
+ should_trigger = any(
75
+ value in rule["contains"] for value in detected_values
76
+ )
77
+ elif rule["type"] == "all":
78
+ should_trigger = all(
79
+ value in rule["contains"] for value in detected_values
80
+ )
81
+
82
+ if should_trigger:
83
+ if rule["reason"]:
84
+ # message = rule['action'] + f' Reason: {result.reason}'
85
+ message = rule["action"]
86
+ reason = result.reason
87
+ else:
88
+ message = rule["action"]
89
+ return rule["metric"], True, message, reason
90
+
91
+ return rule["metric"], False, None, None
92
+
93
+ def _process_rules_batch(
94
+ self, rules: List[Dict], test_case: TestCase, remaining_time: float
95
+ ) -> Tuple[List[str], List[str], List[str]]:
96
+ """
97
+ Process a batch of rules in parallel
98
+
99
+ Args:
100
+ rules: List of rules to process
101
+ test_case: Test case to evaluate
102
+ remaining_time: Time remaining for processing
103
+
104
+ Returns:
105
+ Tuple[List[str], List[str], List[str]]:
106
+ (failure_messages, completed_rules, uncompleted_rules)
107
+ """
108
+ # print(f"\nProcessing batch of {len(rules)} rules")
109
+ # batch_start = time.time()
110
+
111
+ # Submit all rules to the thread pool
112
+ future_to_rule = {
113
+ self.executor.submit(self._check_rule_sync, rule, test_case): rule["metric"]
114
+ for rule in rules
115
+ }
116
+
117
+ completed_rules = []
118
+ uncompleted_rules = [rule["metric"] for rule in rules]
119
+ failure_messages = []
120
+ failure_reasons = []
121
+ failed_rule = None
122
+ try:
123
+ # Wait for futures to complete with timeout
124
+ for future in as_completed(future_to_rule, timeout=remaining_time):
125
+ rule_name = future_to_rule[future]
126
+ try:
127
+ metric, triggered, message, reason = future.result()
128
+ # Update tracking lists
129
+ completed_rules.append(metric)
130
+ uncompleted_rules.remove(metric)
131
+
132
+ if triggered:
133
+ failure_messages.append(message)
134
+ failure_reasons.append(reason)
135
+ failed_rule = rule_name
136
+ # Cancel remaining futures if a rule fails
137
+ for f in future_to_rule:
138
+ if not f.done():
139
+ f.cancel()
140
+ break
141
+
142
+ except Exception as e:
143
+ print(f"Error processing rule {rule_name}: {e}")
144
+
145
+ except TimeoutError:
146
+ print(
147
+ f"Timeout reached. {len(completed_rules)} rules completed, "
148
+ f"{len(uncompleted_rules)} rules incomplete"
149
+ )
150
+
151
+ finally:
152
+ # Cancel any remaining futures
153
+ for future in future_to_rule:
154
+ if not future.done():
155
+ future.cancel()
156
+
157
+ # batch_end = time.time()
158
+ # print(f"Batch processing completed in {batch_end - batch_start:.2f}s\n")
159
+
160
+ return (
161
+ failure_messages,
162
+ completed_rules,
163
+ uncompleted_rules,
164
+ failure_reasons,
165
+ failed_rule,
166
+ )
167
+
168
+ def protect(
169
+ self,
170
+ inputs: str,
171
+ protect_rules: List[Dict],
172
+ action: str = "Response cannot be generated as the input fails the checks",
173
+ reason: bool = False,
174
+ timeout: int = 30000,
175
+ ) -> List[str]:
176
+ """
177
+ Evaluate input strings against protection rules
178
+
179
+ Args:
180
+ inputs: Single string or list of strings to evaluate
181
+ protect_rules: List of protection rule dictionaries. Each rule must contain:
182
+ - metric: str, name of the metric to evaluate (e.g., 'Toxicity', 'Tone', 'Sexism')
183
+ - contains: List[str], values to check for in the evaluation results
184
+ - type: str, either 'any' or 'all'
185
+ - action: str, message to return when rule is triggered
186
+ - reason: bool (optional), whether to include the evaluation reason in the message
187
+ timeout: Optional timeout for evaluations
188
+
189
+ Returns:
190
+ List of protection messages for failed rules or ["All checks passed"] if no rules are triggered
191
+
192
+ Raises:
193
+ ValueError: If inputs or protect_rules don't match the required structure
194
+ TypeError: If inputs contains non-string objects
195
+ """
196
+ # Validate inputs
197
+ if inputs is None:
198
+ raise ValueError("inputs cannot be None")
199
+
200
+ if not isinstance(inputs, str):
201
+ raise TypeError(f"inputs must be a string, got {type(inputs)}")
202
+
203
+ # Convert single string to list
204
+ if isinstance(inputs, str):
205
+ inputs = [inputs]
206
+
207
+ if not inputs:
208
+ raise ValueError("inputs cannot be empty")
209
+
210
+ # Validate each input is a non-empty string
211
+ for i, input_text in enumerate(inputs):
212
+ if not isinstance(input_text, str):
213
+ raise TypeError(
214
+ f"Input at index {i} must be a string, got {type(input_text)}"
215
+ )
216
+ if not input_text.strip():
217
+ raise ValueError(f"Input at index {i} cannot be empty or whitespace")
218
+
219
+ # Convert inputs to TestCase instances
220
+ test_cases = [TestCase(input=input_text) for input_text in inputs]
221
+
222
+ # Validate protect_rules
223
+ if not isinstance(protect_rules, list):
224
+ raise TypeError("protect_rules must be a list")
225
+
226
+ if not protect_rules:
227
+ raise ValueError("protect_rules cannot be empty")
228
+
229
+ valid_metrics = set(self.metric_map.keys())
230
+ valid_types = {"any", "all"}
231
+
232
+ for i, rule in enumerate(protect_rules):
233
+
234
+ if not isinstance(rule, dict):
235
+ raise TypeError(f"Rule at index {i} must be a dictionary")
236
+
237
+ # Check required keys
238
+ required_keys = {"metric"}
239
+ missing_keys = required_keys - set(rule.keys())
240
+ if missing_keys:
241
+ raise ValueError(
242
+ f"Rule at index {i} is missing required keys: {missing_keys}"
243
+ )
244
+
245
+ if rule["metric"] == "Tone":
246
+ required_keys_tone = {"metric", "contains"}
247
+ missing_keys_tone = required_keys_tone - set(rule.keys())
248
+ if missing_keys_tone:
249
+ raise ValueError(
250
+ f"Rule at index {i} is missing required keys: {missing_keys_tone}"
251
+ )
252
+
253
+ # Check for invalid keys
254
+ invalid_keys = set(rule.keys()) - required_keys_tone
255
+ if "type" in rule.keys():
256
+ invalid_keys -= {"type"}
257
+ if invalid_keys:
258
+ raise ValueError(
259
+ f"Invalid key(s) found in rule at index: {invalid_keys}. Valid keys are: {required_keys_tone}"
260
+ )
261
+ else:
262
+ # required_keys_tone = {'metric'}
263
+ # missing_keys_tone = required_keys_tone - set(rule.keys())
264
+ # if missing_keys_tone:
265
+ # raise ValueError(f"Rule at index {i} is missing required keys: {missing_keys_tone}")
266
+
267
+ # Check for invalid keys
268
+ invalid_keys = set(rule.keys()) - required_keys
269
+ if invalid_keys:
270
+ raise ValueError(
271
+ f"Invalid key(s) found in rule at index: {invalid_keys}. Valid keys are: {required_keys}"
272
+ )
273
+
274
+ # required_keys_tone = {'metric', 'contains'}
275
+
276
+ # Validate metric
277
+ if rule["metric"] not in valid_metrics:
278
+ raise ValueError(
279
+ f"Invalid metric '{rule['metric']}' at. "
280
+ f"Valid metrics are: {list(valid_metrics)}"
281
+ )
282
+
283
+ # Validate contains and type only for Tone metric
284
+ if rule["metric"] == "Tone":
285
+ if "contains" not in rule:
286
+ raise ValueError("'contains' is required for Tone metric")
287
+ if not isinstance(rule["contains"], list):
288
+ raise TypeError("'contains' in Tone must be a list")
289
+ if not rule["contains"]:
290
+ raise ValueError("'contains' cannot be empty for Tone metric")
291
+
292
+ if "type" not in rule:
293
+ rule["type"] = "any"
294
+ if rule["type"] not in valid_types:
295
+ raise ValueError(
296
+ f"Invalid type '{rule['type']}' at index {i}. "
297
+ f"Must be one of: {valid_types}"
298
+ )
299
+ else:
300
+ # For non-Tone metrics, these fields should not be specified
301
+ if "contains" in rule:
302
+ raise ValueError(
303
+ f"'contains' should not be specified for {rule['metric']} metric"
304
+ )
305
+ if "type" in rule:
306
+ raise ValueError(
307
+ f"'type' should not be specified for {rule['metric']} metric"
308
+ )
309
+
310
+ # Set default values
311
+ rule["contains"] = ["Failed"]
312
+ rule["type"] = "any"
313
+
314
+ # Validate type
315
+ if "type" not in rule:
316
+ rule["type"] = "any"
317
+
318
+ if rule["type"] not in valid_types:
319
+ raise ValueError(
320
+ f"Invalid type '{rule['type']}' at index {i}. "
321
+ f"Must be one of: {valid_types}"
322
+ )
323
+
324
+ # Validate action
325
+ if "action" not in rule:
326
+ rule["action"] = action
327
+
328
+ # Set default for reason if not provided
329
+ if "reason" in rule:
330
+ raise ValueError("reason cannot be defined in the rule")
331
+ if "reason" not in rule:
332
+ rule["reason"] = reason
333
+
334
+ # results = []
335
+ BATCH_SIZE = 5 # Maximum number of concurrent rule checks
336
+ if len(protect_rules) < BATCH_SIZE:
337
+ BATCH_SIZE = len(protect_rules)
338
+
339
+ total_timeout = timeout / 1000 # in seconds
340
+ start_time = time.time()
341
+
342
+ all_failure_messages = []
343
+ all_completed_rules = []
344
+ all_uncompleted_rules = []
345
+ all_failure_reasons = []
346
+ # try:
347
+ bool_check_fail = False
348
+ for test_case in test_cases:
349
+ for i in range(0, len(protect_rules), BATCH_SIZE):
350
+ # Calculate remaining time
351
+ elapsed_time = time.time() - start_time
352
+ remaining_time = max(0, total_timeout - elapsed_time)
353
+
354
+ if remaining_time <= 0:
355
+ # Add remaining rules to uncompleted list
356
+ remaining_rules = [rule["metric"] for rule in protect_rules[i:]]
357
+ all_uncompleted_rules.extend(remaining_rules)
358
+ break
359
+
360
+ rules_batch = protect_rules[i : i + BATCH_SIZE]
361
+ (
362
+ messages,
363
+ completed,
364
+ uncompleted,
365
+ failure_reasons,
366
+ failed_rule,
367
+ ) = self._process_rules_batch(rules_batch, test_case, remaining_time)
368
+
369
+ all_completed_rules.extend(completed)
370
+ all_uncompleted_rules.extend(uncompleted)
371
+ all_failure_reasons.extend(failure_reasons)
372
+ if messages:
373
+ all_failure_messages.extend(messages)
374
+ bool_check_fail = True
375
+ break
376
+
377
+ ans = {
378
+ "status": "failed" if all_failure_messages else "passed",
379
+ "completed_rules": all_completed_rules,
380
+ "uncompleted_rules": all_uncompleted_rules,
381
+ "failed_rule": failed_rule,
382
+ "messages": all_failure_messages[0]
383
+ if all_failure_messages
384
+ else "All checks passed",
385
+ "reasons": all_failure_reasons[0]
386
+ if all_failure_reasons
387
+ else "All checks passed",
388
+ "time_taken": elapsed_time,
389
+ }
390
+
391
+ if len(ans["uncompleted_rules"]) == len(protect_rules):
392
+ ans["reason"] = "No checks completed"
393
+
394
+ if bool_check_fail:
395
+ ans["status"] = "failed"
396
+ else:
397
+ ans["status"] = "passed"
398
+ # ans['messages'] = inputs
399
+
400
+ if ans["status"] == "passed":
401
+ ans["messages"] = inputs[0]
402
+
403
+ return ans
@@ -320,3 +320,19 @@ class BiasDetection(EvalTemplate):
320
320
 
321
321
  class LLMFunctionCalling(EvalTemplate):
322
322
  eval_id = "72"
323
+
324
+
325
+ class PromptPerplexity(EvalTemplate):
326
+ eval_id = "6"
327
+
328
+
329
+ class ChunkAttribution(EvalTemplate):
330
+ eval_id = "11"
331
+
332
+
333
+ class ChunkUtilization(EvalTemplate):
334
+ eval_id = "12"
335
+
336
+
337
+ class ScoreEval(EvalTemplate):
338
+ eval_id = "65"
@@ -37,4 +37,4 @@ class Routes(str, Enum):
37
37
 
38
38
  dataset_add_run_prompt_column = "model-hub/develops/add_run_prompt_column/"
39
39
  dataset_add_evaluation = "model-hub/develops/{dataset_id}/add_user_eval/"
40
- dataset_optimization_create = "model-hub/optimisation/create/"
40
+ dataset_optimization_create = "model-hub/optimisation/create/"
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "futureagi"
3
- version = "0.4.2"
3
+ version = "0.4.3"
4
4
  description = "We help GenAI teams maintain high-accuracy for their Models in production."
5
5
  authors = ["Future AGI <no-reply@futureagi.com>"]
6
6
  readme = "README.md"
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes