futureagi 0.3.1__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {futureagi-0.3.1 → futureagi-0.3.3}/PKG-INFO +1 -1
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/api/auth.py +2 -2
- futureagi-0.3.3/fi/evals/__init__.py +122 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/evals/evaluator.py +5 -1
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/evals/templates.py +551 -462
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/evals/types.py +12 -7
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/conversational_test_case.py +9 -8
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/general.py +4 -1
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/errors.py +50 -43
- {futureagi-0.3.1 → futureagi-0.3.3}/pyproject.toml +1 -1
- futureagi-0.3.1/fi/evals/__init__.py +0 -148
- {futureagi-0.3.1 → futureagi-0.3.3}/LICENSE.md +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/README.md +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/__init__.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/api/__init__.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/api/types.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/client.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/integrations/__init__.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/integrations/providers/__init__.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/integrations/providers/types.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/__init__.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/testcases/utils.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/__init__.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/constants.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/executor.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/logging.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/routes.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/types.py +0 -0
- {futureagi-0.3.1 → futureagi-0.3.3}/fi/utils/utils.py +0 -0
|
@@ -16,7 +16,7 @@ from fi.utils.constants import (
|
|
|
16
16
|
DEFAULT_TIMEOUT,
|
|
17
17
|
SECRET_KEY_ENVVAR_NAME,
|
|
18
18
|
)
|
|
19
|
-
from fi.utils.errors import
|
|
19
|
+
from fi.utils.errors import MissingAuthError
|
|
20
20
|
from fi.utils.executor import BoundedExecutor
|
|
21
21
|
|
|
22
22
|
T = TypeVar("T")
|
|
@@ -118,7 +118,7 @@ class APIKeyAuth(HttpClient):
|
|
|
118
118
|
SECRET_KEY_ENVVAR_NAME
|
|
119
119
|
)
|
|
120
120
|
if self._fi_api_key is None or self._fi_secret_key is None:
|
|
121
|
-
raise
|
|
121
|
+
raise MissingAuthError(self._fi_api_key, self._fi_secret_key)
|
|
122
122
|
|
|
123
123
|
super().__init__(
|
|
124
124
|
base_url=fi_base_url,
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
from fi.evals.evaluator import EvalClient # noqa: F401
|
|
2
|
+
from fi.evals.templates import (
|
|
3
|
+
PII,
|
|
4
|
+
AnswerRelevance,
|
|
5
|
+
AnswerSimilarity,
|
|
6
|
+
ChunkAttribution,
|
|
7
|
+
ChunkUtilization,
|
|
8
|
+
Completeness,
|
|
9
|
+
Contains,
|
|
10
|
+
ContainsAll,
|
|
11
|
+
ContainsAny,
|
|
12
|
+
ContainsNone,
|
|
13
|
+
ContentModeration,
|
|
14
|
+
ContextAdherence,
|
|
15
|
+
ContextRelevance,
|
|
16
|
+
ContextRetrieval,
|
|
17
|
+
ContextSimilarity,
|
|
18
|
+
ContextSufficiency,
|
|
19
|
+
ConversationCoherence,
|
|
20
|
+
ConversationResolution,
|
|
21
|
+
Correctness,
|
|
22
|
+
CulturalSensitivity,
|
|
23
|
+
DataPrivacyCompliance,
|
|
24
|
+
Deterministic,
|
|
25
|
+
EndsWith,
|
|
26
|
+
Equals,
|
|
27
|
+
FactualAccuracy,
|
|
28
|
+
GradingCriteria,
|
|
29
|
+
Groundedness,
|
|
30
|
+
ImageInputOutput,
|
|
31
|
+
ImageInstruction,
|
|
32
|
+
InstructionAdherence,
|
|
33
|
+
IsJson,
|
|
34
|
+
LegalCompliance,
|
|
35
|
+
LengthLessThan,
|
|
36
|
+
LLMFunctionCalling,
|
|
37
|
+
NoInvalidLinks,
|
|
38
|
+
NotGibberishText,
|
|
39
|
+
Output,
|
|
40
|
+
PromptAdherence,
|
|
41
|
+
PromptInjection,
|
|
42
|
+
PromptPerplexity,
|
|
43
|
+
RagasAnswerCorrectness,
|
|
44
|
+
RagasCoherence,
|
|
45
|
+
RagasConciseness,
|
|
46
|
+
RagasContextPrecision,
|
|
47
|
+
RagasContextRecall,
|
|
48
|
+
RagasContextRelevancy,
|
|
49
|
+
RagasFaithfulness,
|
|
50
|
+
RagasHarmfulness,
|
|
51
|
+
Ranking,
|
|
52
|
+
ReasoningChain,
|
|
53
|
+
Regex,
|
|
54
|
+
SafeForWorkText,
|
|
55
|
+
Sexist,
|
|
56
|
+
SummaryAccuracy,
|
|
57
|
+
SummaryQuality,
|
|
58
|
+
Tone,
|
|
59
|
+
Toxicity,
|
|
60
|
+
TranslationAccuracy,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"AnswerRelevance",
|
|
65
|
+
"AnswerSimilarity",
|
|
66
|
+
"ChunkAttribution",
|
|
67
|
+
"ChunkUtilization",
|
|
68
|
+
"Completeness",
|
|
69
|
+
"Contains",
|
|
70
|
+
"ContainsAll",
|
|
71
|
+
"ContainsAny",
|
|
72
|
+
"ContainsNone",
|
|
73
|
+
"ContextAdherence",
|
|
74
|
+
"ContextRetrieval",
|
|
75
|
+
"ContextRelevance",
|
|
76
|
+
"ContextSimilarity",
|
|
77
|
+
"ContextSufficiency",
|
|
78
|
+
"ContentModeration",
|
|
79
|
+
"ConversationCoherence",
|
|
80
|
+
"ConversationResolution",
|
|
81
|
+
"Correctness",
|
|
82
|
+
"CulturalSensitivity",
|
|
83
|
+
"DataPrivacyCompliance",
|
|
84
|
+
"Deterministic",
|
|
85
|
+
"EndsWith",
|
|
86
|
+
"Equals",
|
|
87
|
+
"FactualAccuracy",
|
|
88
|
+
"GradingCriteria",
|
|
89
|
+
"Groundedness",
|
|
90
|
+
"ImageInputOutput",
|
|
91
|
+
"ImageInstruction",
|
|
92
|
+
"InstructionAdherence",
|
|
93
|
+
"IsJson",
|
|
94
|
+
"LegalCompliance",
|
|
95
|
+
"LengthLessThan",
|
|
96
|
+
"LLMFunctionCalling",
|
|
97
|
+
"NoInvalidLinks",
|
|
98
|
+
"NotGibberishText",
|
|
99
|
+
"Output",
|
|
100
|
+
"PII",
|
|
101
|
+
"PromptAdherence",
|
|
102
|
+
"PromptInjection",
|
|
103
|
+
"PromptPerplexity",
|
|
104
|
+
"RagasAnswerCorrectness",
|
|
105
|
+
"RagasCoherence",
|
|
106
|
+
"RagasConciseness",
|
|
107
|
+
"RagasContextPrecision",
|
|
108
|
+
"RagasContextRecall",
|
|
109
|
+
"RagasContextRelevancy",
|
|
110
|
+
"RagasFaithfulness",
|
|
111
|
+
"RagasHarmfulness",
|
|
112
|
+
"Ranking",
|
|
113
|
+
"ReasoningChain",
|
|
114
|
+
"Regex",
|
|
115
|
+
"SafeForWorkText",
|
|
116
|
+
"Sexist",
|
|
117
|
+
"SummaryAccuracy",
|
|
118
|
+
"SummaryQuality",
|
|
119
|
+
"Tone",
|
|
120
|
+
"Toxicity",
|
|
121
|
+
"TranslationAccuracy",
|
|
122
|
+
]
|
|
@@ -7,6 +7,7 @@ from fi.api.types import HttpMethod, RequestConfig
|
|
|
7
7
|
from fi.evals.templates import EvalTemplate
|
|
8
8
|
from fi.evals.types import BatchRunResult, EvalResult, EvalResultMetric
|
|
9
9
|
from fi.testcases import ConversationalTestCase, LLMTestCase, MLLMTestCase, TestCase
|
|
10
|
+
from fi.utils.errors import InvalidAuthError
|
|
10
11
|
from fi.utils.routes import Routes
|
|
11
12
|
|
|
12
13
|
|
|
@@ -21,6 +22,8 @@ class EvalResponseHandler(ResponseHandler[BatchRunResult]):
|
|
|
21
22
|
def _handle_error(cls, response: Response) -> None:
|
|
22
23
|
if response.status_code == 400:
|
|
23
24
|
response.raise_for_status()
|
|
25
|
+
if response.status_code == 403:
|
|
26
|
+
raise InvalidAuthError()
|
|
24
27
|
|
|
25
28
|
@classmethod
|
|
26
29
|
def convert_to_batch_results(cls, response: Dict[str, Any]) -> BatchRunResult:
|
|
@@ -126,7 +129,8 @@ class EvalClient(APIKeyAuth):
|
|
|
126
129
|
payload = {
|
|
127
130
|
"inputs": [test_case.model_dump() for test_case in inputs],
|
|
128
131
|
"config": {
|
|
129
|
-
eval_object.
|
|
132
|
+
eval_object.eval_id: eval_object.config
|
|
133
|
+
for eval_object in eval_templates
|
|
130
134
|
},
|
|
131
135
|
}
|
|
132
136
|
|