futureagi 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {futureagi-0.3.2 → futureagi-0.3.3}/PKG-INFO +1 -1
- futureagi-0.3.3/fi/evals/__init__.py +122 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/evals/evaluator.py +2 -1
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/evals/templates.py +551 -462
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/evals/types.py +12 -7
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/testcases/conversational_test_case.py +9 -8
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/testcases/general.py +4 -1
- {futureagi-0.3.2 → futureagi-0.3.3}/pyproject.toml +1 -1
- futureagi-0.3.2/fi/evals/__init__.py +0 -148
- {futureagi-0.3.2 → futureagi-0.3.3}/LICENSE.md +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/README.md +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/__init__.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/api/__init__.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/api/auth.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/api/types.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/client.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/integrations/__init__.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/integrations/providers/__init__.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/integrations/providers/types.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/testcases/__init__.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/testcases/llm_test_case.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/testcases/mllm_test_case.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/testcases/utils.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/__init__.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/constants.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/errors.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/executor.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/logging.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/routes.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/types.py +0 -0
- {futureagi-0.3.2 → futureagi-0.3.3}/fi/utils/utils.py +0 -0
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
from fi.evals.evaluator import EvalClient # noqa: F401
|
|
2
|
+
from fi.evals.templates import (
|
|
3
|
+
PII,
|
|
4
|
+
AnswerRelevance,
|
|
5
|
+
AnswerSimilarity,
|
|
6
|
+
ChunkAttribution,
|
|
7
|
+
ChunkUtilization,
|
|
8
|
+
Completeness,
|
|
9
|
+
Contains,
|
|
10
|
+
ContainsAll,
|
|
11
|
+
ContainsAny,
|
|
12
|
+
ContainsNone,
|
|
13
|
+
ContentModeration,
|
|
14
|
+
ContextAdherence,
|
|
15
|
+
ContextRelevance,
|
|
16
|
+
ContextRetrieval,
|
|
17
|
+
ContextSimilarity,
|
|
18
|
+
ContextSufficiency,
|
|
19
|
+
ConversationCoherence,
|
|
20
|
+
ConversationResolution,
|
|
21
|
+
Correctness,
|
|
22
|
+
CulturalSensitivity,
|
|
23
|
+
DataPrivacyCompliance,
|
|
24
|
+
Deterministic,
|
|
25
|
+
EndsWith,
|
|
26
|
+
Equals,
|
|
27
|
+
FactualAccuracy,
|
|
28
|
+
GradingCriteria,
|
|
29
|
+
Groundedness,
|
|
30
|
+
ImageInputOutput,
|
|
31
|
+
ImageInstruction,
|
|
32
|
+
InstructionAdherence,
|
|
33
|
+
IsJson,
|
|
34
|
+
LegalCompliance,
|
|
35
|
+
LengthLessThan,
|
|
36
|
+
LLMFunctionCalling,
|
|
37
|
+
NoInvalidLinks,
|
|
38
|
+
NotGibberishText,
|
|
39
|
+
Output,
|
|
40
|
+
PromptAdherence,
|
|
41
|
+
PromptInjection,
|
|
42
|
+
PromptPerplexity,
|
|
43
|
+
RagasAnswerCorrectness,
|
|
44
|
+
RagasCoherence,
|
|
45
|
+
RagasConciseness,
|
|
46
|
+
RagasContextPrecision,
|
|
47
|
+
RagasContextRecall,
|
|
48
|
+
RagasContextRelevancy,
|
|
49
|
+
RagasFaithfulness,
|
|
50
|
+
RagasHarmfulness,
|
|
51
|
+
Ranking,
|
|
52
|
+
ReasoningChain,
|
|
53
|
+
Regex,
|
|
54
|
+
SafeForWorkText,
|
|
55
|
+
Sexist,
|
|
56
|
+
SummaryAccuracy,
|
|
57
|
+
SummaryQuality,
|
|
58
|
+
Tone,
|
|
59
|
+
Toxicity,
|
|
60
|
+
TranslationAccuracy,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
__all__ = [
|
|
64
|
+
"AnswerRelevance",
|
|
65
|
+
"AnswerSimilarity",
|
|
66
|
+
"ChunkAttribution",
|
|
67
|
+
"ChunkUtilization",
|
|
68
|
+
"Completeness",
|
|
69
|
+
"Contains",
|
|
70
|
+
"ContainsAll",
|
|
71
|
+
"ContainsAny",
|
|
72
|
+
"ContainsNone",
|
|
73
|
+
"ContextAdherence",
|
|
74
|
+
"ContextRetrieval",
|
|
75
|
+
"ContextRelevance",
|
|
76
|
+
"ContextSimilarity",
|
|
77
|
+
"ContextSufficiency",
|
|
78
|
+
"ContentModeration",
|
|
79
|
+
"ConversationCoherence",
|
|
80
|
+
"ConversationResolution",
|
|
81
|
+
"Correctness",
|
|
82
|
+
"CulturalSensitivity",
|
|
83
|
+
"DataPrivacyCompliance",
|
|
84
|
+
"Deterministic",
|
|
85
|
+
"EndsWith",
|
|
86
|
+
"Equals",
|
|
87
|
+
"FactualAccuracy",
|
|
88
|
+
"GradingCriteria",
|
|
89
|
+
"Groundedness",
|
|
90
|
+
"ImageInputOutput",
|
|
91
|
+
"ImageInstruction",
|
|
92
|
+
"InstructionAdherence",
|
|
93
|
+
"IsJson",
|
|
94
|
+
"LegalCompliance",
|
|
95
|
+
"LengthLessThan",
|
|
96
|
+
"LLMFunctionCalling",
|
|
97
|
+
"NoInvalidLinks",
|
|
98
|
+
"NotGibberishText",
|
|
99
|
+
"Output",
|
|
100
|
+
"PII",
|
|
101
|
+
"PromptAdherence",
|
|
102
|
+
"PromptInjection",
|
|
103
|
+
"PromptPerplexity",
|
|
104
|
+
"RagasAnswerCorrectness",
|
|
105
|
+
"RagasCoherence",
|
|
106
|
+
"RagasConciseness",
|
|
107
|
+
"RagasContextPrecision",
|
|
108
|
+
"RagasContextRecall",
|
|
109
|
+
"RagasContextRelevancy",
|
|
110
|
+
"RagasFaithfulness",
|
|
111
|
+
"RagasHarmfulness",
|
|
112
|
+
"Ranking",
|
|
113
|
+
"ReasoningChain",
|
|
114
|
+
"Regex",
|
|
115
|
+
"SafeForWorkText",
|
|
116
|
+
"Sexist",
|
|
117
|
+
"SummaryAccuracy",
|
|
118
|
+
"SummaryQuality",
|
|
119
|
+
"Tone",
|
|
120
|
+
"Toxicity",
|
|
121
|
+
"TranslationAccuracy",
|
|
122
|
+
]
|
|
@@ -129,7 +129,8 @@ class EvalClient(APIKeyAuth):
|
|
|
129
129
|
payload = {
|
|
130
130
|
"inputs": [test_case.model_dump() for test_case in inputs],
|
|
131
131
|
"config": {
|
|
132
|
-
eval_object.
|
|
132
|
+
eval_object.eval_id: eval_object.config
|
|
133
|
+
for eval_object in eval_templates
|
|
133
134
|
},
|
|
134
135
|
}
|
|
135
136
|
|