futureagi 0.3.1__tar.gz → 0.3.3__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (31) hide show
  1. {futureagi-0.3.1 → futureagi-0.3.3}/PKG-INFO +1 -1
  2. {futureagi-0.3.1 → futureagi-0.3.3}/fi/api/auth.py +2 -2
  3. futureagi-0.3.3/fi/evals/__init__.py +122 -0
  4. {futureagi-0.3.1 → futureagi-0.3.3}/fi/evals/evaluator.py +5 -1
  5. {futureagi-0.3.1 → futureagi-0.3.3}/fi/evals/templates.py +551 -462
  6. {futureagi-0.3.1 → futureagi-0.3.3}/fi/evals/types.py +12 -7
  7. {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/conversational_test_case.py +9 -8
  8. {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/general.py +4 -1
  9. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/errors.py +50 -43
  10. {futureagi-0.3.1 → futureagi-0.3.3}/pyproject.toml +1 -1
  11. futureagi-0.3.1/fi/evals/__init__.py +0 -148
  12. {futureagi-0.3.1 → futureagi-0.3.3}/LICENSE.md +0 -0
  13. {futureagi-0.3.1 → futureagi-0.3.3}/README.md +0 -0
  14. {futureagi-0.3.1 → futureagi-0.3.3}/fi/__init__.py +0 -0
  15. {futureagi-0.3.1 → futureagi-0.3.3}/fi/api/__init__.py +0 -0
  16. {futureagi-0.3.1 → futureagi-0.3.3}/fi/api/types.py +0 -0
  17. {futureagi-0.3.1 → futureagi-0.3.3}/fi/client.py +0 -0
  18. {futureagi-0.3.1 → futureagi-0.3.3}/fi/integrations/__init__.py +0 -0
  19. {futureagi-0.3.1 → futureagi-0.3.3}/fi/integrations/providers/__init__.py +0 -0
  20. {futureagi-0.3.1 → futureagi-0.3.3}/fi/integrations/providers/types.py +0 -0
  21. {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/__init__.py +0 -0
  22. {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/llm_test_case.py +0 -0
  23. {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/mllm_test_case.py +0 -0
  24. {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/utils.py +0 -0
  25. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/__init__.py +0 -0
  26. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/constants.py +0 -0
  27. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/executor.py +0 -0
  28. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/logging.py +0 -0
  29. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/routes.py +0 -0
  30. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/types.py +0 -0
  31. {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/utils.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: futureagi
3
- Version: 0.3.1
3
+ Version: 0.3.3
4
4
  Summary: We help GenAI teams maintain high-accuracy for their Models in production.
5
5
  Author: Future AGI
6
6
  Author-email: no-reply@futureagi.com
@@ -16,7 +16,7 @@ from fi.utils.constants import (
16
16
  DEFAULT_TIMEOUT,
17
17
  SECRET_KEY_ENVVAR_NAME,
18
18
  )
19
- from fi.utils.errors import AuthError
19
+ from fi.utils.errors import MissingAuthError
20
20
  from fi.utils.executor import BoundedExecutor
21
21
 
22
22
  T = TypeVar("T")
@@ -118,7 +118,7 @@ class APIKeyAuth(HttpClient):
118
118
  SECRET_KEY_ENVVAR_NAME
119
119
  )
120
120
  if self._fi_api_key is None or self._fi_secret_key is None:
121
- raise AuthError(self._fi_api_key, self._fi_secret_key)
121
+ raise MissingAuthError(self._fi_api_key, self._fi_secret_key)
122
122
 
123
123
  super().__init__(
124
124
  base_url=fi_base_url,
@@ -0,0 +1,122 @@
1
+ from fi.evals.evaluator import EvalClient # noqa: F401
2
+ from fi.evals.templates import (
3
+ PII,
4
+ AnswerRelevance,
5
+ AnswerSimilarity,
6
+ ChunkAttribution,
7
+ ChunkUtilization,
8
+ Completeness,
9
+ Contains,
10
+ ContainsAll,
11
+ ContainsAny,
12
+ ContainsNone,
13
+ ContentModeration,
14
+ ContextAdherence,
15
+ ContextRelevance,
16
+ ContextRetrieval,
17
+ ContextSimilarity,
18
+ ContextSufficiency,
19
+ ConversationCoherence,
20
+ ConversationResolution,
21
+ Correctness,
22
+ CulturalSensitivity,
23
+ DataPrivacyCompliance,
24
+ Deterministic,
25
+ EndsWith,
26
+ Equals,
27
+ FactualAccuracy,
28
+ GradingCriteria,
29
+ Groundedness,
30
+ ImageInputOutput,
31
+ ImageInstruction,
32
+ InstructionAdherence,
33
+ IsJson,
34
+ LegalCompliance,
35
+ LengthLessThan,
36
+ LLMFunctionCalling,
37
+ NoInvalidLinks,
38
+ NotGibberishText,
39
+ Output,
40
+ PromptAdherence,
41
+ PromptInjection,
42
+ PromptPerplexity,
43
+ RagasAnswerCorrectness,
44
+ RagasCoherence,
45
+ RagasConciseness,
46
+ RagasContextPrecision,
47
+ RagasContextRecall,
48
+ RagasContextRelevancy,
49
+ RagasFaithfulness,
50
+ RagasHarmfulness,
51
+ Ranking,
52
+ ReasoningChain,
53
+ Regex,
54
+ SafeForWorkText,
55
+ Sexist,
56
+ SummaryAccuracy,
57
+ SummaryQuality,
58
+ Tone,
59
+ Toxicity,
60
+ TranslationAccuracy,
61
+ )
62
+
63
+ __all__ = [
64
+ "AnswerRelevance",
65
+ "AnswerSimilarity",
66
+ "ChunkAttribution",
67
+ "ChunkUtilization",
68
+ "Completeness",
69
+ "Contains",
70
+ "ContainsAll",
71
+ "ContainsAny",
72
+ "ContainsNone",
73
+ "ContextAdherence",
74
+ "ContextRetrieval",
75
+ "ContextRelevance",
76
+ "ContextSimilarity",
77
+ "ContextSufficiency",
78
+ "ContentModeration",
79
+ "ConversationCoherence",
80
+ "ConversationResolution",
81
+ "Correctness",
82
+ "CulturalSensitivity",
83
+ "DataPrivacyCompliance",
84
+ "Deterministic",
85
+ "EndsWith",
86
+ "Equals",
87
+ "FactualAccuracy",
88
+ "GradingCriteria",
89
+ "Groundedness",
90
+ "ImageInputOutput",
91
+ "ImageInstruction",
92
+ "InstructionAdherence",
93
+ "IsJson",
94
+ "LegalCompliance",
95
+ "LengthLessThan",
96
+ "LLMFunctionCalling",
97
+ "NoInvalidLinks",
98
+ "NotGibberishText",
99
+ "Output",
100
+ "PII",
101
+ "PromptAdherence",
102
+ "PromptInjection",
103
+ "PromptPerplexity",
104
+ "RagasAnswerCorrectness",
105
+ "RagasCoherence",
106
+ "RagasConciseness",
107
+ "RagasContextPrecision",
108
+ "RagasContextRecall",
109
+ "RagasContextRelevancy",
110
+ "RagasFaithfulness",
111
+ "RagasHarmfulness",
112
+ "Ranking",
113
+ "ReasoningChain",
114
+ "Regex",
115
+ "SafeForWorkText",
116
+ "Sexist",
117
+ "SummaryAccuracy",
118
+ "SummaryQuality",
119
+ "Tone",
120
+ "Toxicity",
121
+ "TranslationAccuracy",
122
+ ]
@@ -7,6 +7,7 @@ from fi.api.types import HttpMethod, RequestConfig
7
7
  from fi.evals.templates import EvalTemplate
8
8
  from fi.evals.types import BatchRunResult, EvalResult, EvalResultMetric
9
9
  from fi.testcases import ConversationalTestCase, LLMTestCase, MLLMTestCase, TestCase
10
+ from fi.utils.errors import InvalidAuthError
10
11
  from fi.utils.routes import Routes
11
12
 
12
13
 
@@ -21,6 +22,8 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult]):
21
22
  def _handle_error(cls, response: Response) -> None:
22
23
  if response.status_code == 400:
23
24
  response.raise_for_status()
25
+ if response.status_code == 403:
26
+ raise InvalidAuthError()
24
27
 
25
28
  @classmethod
26
29
  def convert_to_batch_results(cls, response: Dict[str, Any]) -> BatchRunResult:
@@ -126,7 +129,8 @@ class EvalClient(APIKeyAuth):
126
129
  payload = {
127
130
  "inputs": [test_case.model_dump() for test_case in inputs],
128
131
  "config": {
129
- eval_object.name: eval_object.config for eval_object in eval_templates
132
+ eval_object.eval_id: eval_object.config
133
+ for eval_object in eval_templates
130
134
  },
131
135
  }
132
136