fiddler-evals 0.1.1.dev14__tar.gz → 0.2.0rc1__tar.gz

{fiddler_evals-0.1.1.dev14/fiddler_evals.egg-info → fiddler_evals-0.2.0rc1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.1.1.dev14
+Version: 0.2.0rc1
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
 Maintainer-email: Fiddler AI <support@fiddler.ai>
@@ -15,7 +15,6 @@ Requires-Dist: requests<3
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: tqdm
 Requires-Dist: typing-extensions<5,>=4.6.0
-Requires-Dist: pandas>=1.2.5
 Requires-Dist: python-decouple
 Provides-Extra: pandas
 Requires-Dist: pandas>=1.2.5; extra == "pandas"
@@ -101,19 +100,32 @@ dataset.insert(test_cases)
 
 ### 4. Use Built-in Evaluators
 
+**Configure LLM Gateway provider:**
+
+Add an LLM provider via UI (**Settings > LLM Gateway**) to use Fiddler's pre-build LLM-as-a-Judge evaluators.
+LLM-as-a-Judge evaluators require a `model` parameter in the format `{provider}/{model}` (e.g., `openai/gpt-4o`) and an optional `credential` parameter for LLM Gateway authentication.
+
 ```python
 from fiddler_evals.evaluators import (
     AnswerRelevance, Coherence, Conciseness,
-    Toxicity, Sentiment, RegexSearch
+    Sentiment, RegexSearch
 )
 
-# Test individual evaluators
-relevance_evaluator = AnswerRelevance()
+# Test LLM-as-a-Judge evaluators (require model parameter)
+relevance_evaluator = AnswerRelevance(
+    model="openai/gpt-4o",           # Required: LLM Gateway model in {provider}/{model} format
+    credential="my-openai-cred"      # Optional: LLM Gateway credential name
+)
 score = relevance_evaluator.score(
     prompt="What is the capital of France?",
     response="Paris is the capital of France."
 )
 print(f"Score: {score.value} - {score.reasoning}")
+
+# Test other evaluators (no model parameter needed)
+sentiment_evaluator = Sentiment()
+scores = sentiment_evaluator.score(text="This is a helpful response.")
+print("Sentiments:", [f'{score.name}: {score.value}' for score in scores])
 ```
 
 ### 5. Create Custom Evaluators
@@ -199,8 +211,8 @@ def contains_number_evaluator(output: str) -> float:
 
 # Use functions directly in evaluators list
 evaluators = [
-    AnswerRelevance(),
-    Conciseness(),
+    AnswerRelevance(model="openai/gpt-4o", credential="my-openai-cred"),
+    Conciseness(model="openai/gpt-4o", credential="my-openai-cred"),
     word_count_evaluator,        # Function evaluator
     contains_number_evaluator,   # Function evaluator
 ]
@@ -231,9 +243,19 @@ def my_llm_task(inputs: dict, extras: dict, metadata: dict) -> dict:
 
 # Set up evaluators with different configurations
 evaluators = [
-    # Primary evaluation metrics
-    AnswerRelevance(score_name_prefix="primary"),
-    Conciseness(score_name_prefix="primary"),
+    # LLM-as-a-Judge evaluators (require model parameter)
+    AnswerRelevance(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+    Conciseness(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+
+    # Other evaluators
     Sentiment(score_name_prefix="primary"),
 
     # Custom evaluators with specific mappings
@@ -245,13 +267,13 @@ evaluators = [
     # Multiple instances of same evaluator for different fields
     RegexSearch(
         pattern=r"\d+",
-        score_name_prefix="validation",
+        score_name_prefix="question",
         score_name="has_number",
         score_fn_kwargs_mapping={"output": "question"}
     ),
     RegexSearch(
         pattern=r"\d+",
-        score_name_prefix="validation",
+        score_name_prefix="answer",
         score_name="has_number",
         score_fn_kwargs_mapping={"output": "answer"}
     ),
@@ -277,21 +299,22 @@ print(f"Generated {sum(len(result.scores) for result in experiment_result.result
 
 # Results in organized score names:
 # "primary_answer_relevance", "primary_conciseness", "primary_sentiment",
-# "quality_politeness", "validation_has_number" (for question), "validation_has_number" (for answer)
+# "quality_politeness", "question_has_number", "answer_has_number"
 ```
 
 ## Built-in Evaluators
 
-| Evaluator | Purpose | Key Parameters |
-|-----------|---------|----------------|
-| `AnswerRelevance` | Checks if response addresses the question | `prompt`, `response` |
-| `Coherence` | Evaluates logical flow and consistency | `response`, `prompt` |
-| `Conciseness` | Measures response brevity and clarity | `response` |
-| `Toxicity` | Detects harmful or toxic content | `text` |
-| `Sentiment` | Analyzes emotional tone | `text` |
-| `RegexSearch` | Pattern matching for specific formats | `output`, `pattern` |
-| `FTLPromptSafety` | Compute safety scores for prompts | `text` |
-| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | `response`, `context` |
+| Evaluator | Purpose | Constructor Parameters | Score Parameters |
+|-----------|---------|------------------------|------------------|
+| `AnswerRelevance` | Checks if response addresses the question | `model` (required), `credential` (required) | `prompt`, `response` |
+| `Coherence` | Evaluates logical flow and consistency | `model` (required), `credential` (required) | `response`, `prompt` (optional) |
+| `Conciseness` | Measures response brevity and clarity | `model` (required), `credential` (required) | `response` |
+| `Sentiment` | Analyzes emotional tone | - | `text` |
+| `RegexSearch` | Pattern matching for specific formats | `pattern` (required) | `output` |
+| `FTLPromptSafety` | Compute safety scores for prompts | - | `text` |
+| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | - | `response`, `context` |
+
+**Note:** Evaluators marked with `model` and `credential` parameters are LLM-as-a-Judge evaluators that require an LLM Gateway model. The `model` parameter should be in `{provider}/{model}` format (e.g., `openai/gpt-4o`). The `credential` parameter is the name of the LLM Gateway credential for authentication.
 
 ## Data Import Options
 

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/PUBLIC.md

@@ -79,19 +79,32 @@ dataset.insert(test_cases)
 
 ### 4. Use Built-in Evaluators
 
+**Configure LLM Gateway provider:**
+
+Add an LLM provider via UI (**Settings > LLM Gateway**) to use Fiddler's pre-build LLM-as-a-Judge evaluators.
+LLM-as-a-Judge evaluators require a `model` parameter in the format `{provider}/{model}` (e.g., `openai/gpt-4o`) and an optional `credential` parameter for LLM Gateway authentication.
+
 ```python
 from fiddler_evals.evaluators import (
     AnswerRelevance, Coherence, Conciseness,
-    Toxicity, Sentiment, RegexSearch
+    Sentiment, RegexSearch
 )
 
-# Test individual evaluators
-relevance_evaluator = AnswerRelevance()
+# Test LLM-as-a-Judge evaluators (require model parameter)
+relevance_evaluator = AnswerRelevance(
+    model="openai/gpt-4o",           # Required: LLM Gateway model in {provider}/{model} format
+    credential="my-openai-cred"      # Optional: LLM Gateway credential name
+)
 score = relevance_evaluator.score(
     prompt="What is the capital of France?",
     response="Paris is the capital of France."
 )
 print(f"Score: {score.value} - {score.reasoning}")
+
+# Test other evaluators (no model parameter needed)
+sentiment_evaluator = Sentiment()
+scores = sentiment_evaluator.score(text="This is a helpful response.")
+print("Sentiments:", [f'{score.name}: {score.value}' for score in scores])
 ```
 
 ### 5. Create Custom Evaluators
@@ -177,8 +190,8 @@ def contains_number_evaluator(output: str) -> float:
 
 # Use functions directly in evaluators list
 evaluators = [
-    AnswerRelevance(),
-    Conciseness(),
+    AnswerRelevance(model="openai/gpt-4o", credential="my-openai-cred"),
+    Conciseness(model="openai/gpt-4o", credential="my-openai-cred"),
     word_count_evaluator,        # Function evaluator
     contains_number_evaluator,   # Function evaluator
 ]
@@ -209,9 +222,19 @@ def my_llm_task(inputs: dict, extras: dict, metadata: dict) -> dict:
 
 # Set up evaluators with different configurations
 evaluators = [
-    # Primary evaluation metrics
-    AnswerRelevance(score_name_prefix="primary"),
-    Conciseness(score_name_prefix="primary"),
+    # LLM-as-a-Judge evaluators (require model parameter)
+    AnswerRelevance(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+    Conciseness(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+
+    # Other evaluators
     Sentiment(score_name_prefix="primary"),
 
     # Custom evaluators with specific mappings
@@ -223,13 +246,13 @@ evaluators = [
     # Multiple instances of same evaluator for different fields
     RegexSearch(
         pattern=r"\d+",
-        score_name_prefix="validation",
+        score_name_prefix="question",
         score_name="has_number",
         score_fn_kwargs_mapping={"output": "question"}
     ),
     RegexSearch(
         pattern=r"\d+",
-        score_name_prefix="validation",
+        score_name_prefix="answer",
         score_name="has_number",
         score_fn_kwargs_mapping={"output": "answer"}
     ),
@@ -255,21 +278,22 @@ print(f"Generated {sum(len(result.scores) for result in experiment_result.result
 
 # Results in organized score names:
 # "primary_answer_relevance", "primary_conciseness", "primary_sentiment",
-# "quality_politeness", "validation_has_number" (for question), "validation_has_number" (for answer)
+# "quality_politeness", "question_has_number", "answer_has_number"
 ```
 
 ## Built-in Evaluators
 
-| Evaluator | Purpose | Key Parameters |
-|-----------|---------|----------------|
-| `AnswerRelevance` | Checks if response addresses the question | `prompt`, `response` |
-| `Coherence` | Evaluates logical flow and consistency | `response`, `prompt` |
-| `Conciseness` | Measures response brevity and clarity | `response` |
-| `Toxicity` | Detects harmful or toxic content | `text` |
-| `Sentiment` | Analyzes emotional tone | `text` |
-| `RegexSearch` | Pattern matching for specific formats | `output`, `pattern` |
-| `FTLPromptSafety` | Compute safety scores for prompts | `text` |
-| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | `response`, `context` |
+| Evaluator | Purpose | Constructor Parameters | Score Parameters |
+|-----------|---------|------------------------|------------------|
+| `AnswerRelevance` | Checks if response addresses the question | `model` (required), `credential` (required) | `prompt`, `response` |
+| `Coherence` | Evaluates logical flow and consistency | `model` (required), `credential` (required) | `response`, `prompt` (optional) |
+| `Conciseness` | Measures response brevity and clarity | `model` (required), `credential` (required) | `response` |
+| `Sentiment` | Analyzes emotional tone | - | `text` |
+| `RegexSearch` | Pattern matching for specific formats | `pattern` (required) | `output` |
+| `FTLPromptSafety` | Compute safety scores for prompts | - | `text` |
+| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | - | `response`, `context` |
+
+**Note:** Evaluators marked with `model` and `credential` parameters are LLM-as-a-Judge evaluators that require an LLM Gateway model. The `model` parameter should be in `{provider}/{model}` format (e.g., `openai/gpt-4o`). The `credential` parameter is the name of the LLM Gateway credential for authentication.
 
 ## Data Import Options
 

fiddler_evals-0.2.0rc1/fiddler_evals/VERSION

@@ -0,0 +1 @@
+0.2.0rc1

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/fiddler_evals/__init__.py

@@ -20,7 +20,6 @@ from fiddler_evals.evaluators import (
     RegexSearch,
     Sentiment,
     TopicClassification,
-    Toxicity,
 )
 from fiddler_evals.evaluators.base import Evaluator
 from fiddler_evals.evaluators.eval_fn import EvalFn
@@ -55,7 +54,6 @@ __all__ = [
     "AnswerRelevance",
     "Coherence",
     "Conciseness",
-    "Toxicity",
     "Sentiment",
     "RegexSearch",
     "RegexMatch",

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/fiddler_evals/evaluators/__init__.py

@@ -7,7 +7,6 @@ from fiddler_evals.evaluators.ftl_response_faithfulness import FTLResponseFaithf
 from fiddler_evals.evaluators.regex import RegexMatch, RegexSearch
 from fiddler_evals.evaluators.sentiment import Sentiment
 from fiddler_evals.evaluators.topic import TopicClassification
-from fiddler_evals.evaluators.toxicity import Toxicity
 
 __all__ = [
     "RegexSearch",
@@ -17,7 +16,6 @@ __all__ = [
     "Conciseness",
     "FTLPromptSafety",
     "FTLResponseFaithfulness",
-    "Toxicity",
     "Sentiment",
     "TopicClassification",
     "EvalFn",

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/fiddler_evals/evaluators/answer_relevance.py

@@ -1,8 +1,8 @@
-from fiddler_evals.evaluators.base import FiddlerEvaluator
+from fiddler_evals.evaluators.base import FiddlerLLMAAJEvaluator
 from fiddler_evals.pydantic_models.score import Score
 
 
-class AnswerRelevance(FiddlerEvaluator):
+class AnswerRelevance(FiddlerLLMAAJEvaluator):
     """Evaluator to assess how well an answer addresses a given question.
 
     The AnswerRelevance evaluator measures whether an LLM's answer is relevant
@@ -85,8 +85,14 @@ class AnswerRelevance(FiddlerEvaluator):
 
         payload = {
             "evaluator_name": self.name,
-            "parameters": {},
-            "inputs": {"prompt": prompt, "response": response},
+            "parameters": {
+                "model": self.model,
+                "credential": self.credential,
+            },
+            "inputs": {
+                "prompt": prompt,
+                "response": response,
+            },
         }
 
         return self._parse_scores(data=self.make_call(payload))[0]

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/fiddler_evals/evaluators/base.py

@@ -243,3 +243,26 @@ class FiddlerEvaluator(Evaluator, ABC):
             scores.append(score)
 
         return scores
+
+
+class FiddlerLLMAAJEvaluator(FiddlerEvaluator, ABC):
+    """Base class for LLMAAJ evaluators that use Fiddler's evaluator API."""
+
+    def __init__(
+        self, model: str, credential: str | None = None, **kwargs: Any
+    ) -> None:
+        """Initialize the LLMAAJ evaluator with model and credential.
+
+        Args:
+            model (str): LLM Gateway model name in `{provider}/{model}` format.
+                E.g., `openai/gpt-4o`
+            credential (str): Name of the LLM Gateway credential for the above provider.
+            **kwargs: Additional keyword arguments
+        """
+        super().__init__(**kwargs)
+
+        if not model:
+            raise ValueError("model is required for LLMAAJ based evaluators")
+
+        self.model = model
+        self.credential = credential

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/fiddler_evals/evaluators/coherence.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
-from fiddler_evals.evaluators.base import FiddlerEvaluator
+from fiddler_evals.evaluators.base import FiddlerLLMAAJEvaluator
 from fiddler_evals.pydantic_models.score import Score
 
 
-class Coherence(FiddlerEvaluator):
+class Coherence(FiddlerLLMAAJEvaluator):
     """Evaluator to assess the coherence and logical flow of a response.
 
     The Coherence evaluator measures whether a response is well-structured, logically
@@ -58,7 +58,8 @@ class Coherence(FiddlerEvaluator):
 
         # Incoherent response
         incoherent_score = evaluator.score(
-            response="The sky is blue. I like pizza. Quantum physics is complex. Let's go shopping."
+            prompt="Explain the process of making coffee",
+            response="The sky is blue. I like pizza. Quantum physics is complex. Let's go shopping.",
         )
         print(f"Coherence: {incoherent_score.value}")  # 0.0
 
@@ -83,28 +84,33 @@ class Coherence(FiddlerEvaluator):
 
     name = "coherence"
 
-    def score(self, response: str, prompt: str | None = None) -> Score:  # pylint: disable=arguments-differ
+    def score(self, prompt: str, response: str) -> Score:  # pylint: disable=arguments-differ
         """Score the coherence of a response.
 
         Args:
+            prompt (str): The original prompt that generated the response.
             response (str): The response to evaluate for coherence.
-            prompt (str, optional): The original prompt that generated the response.
 
         Returns:
             Score: A Score object for coherence assessment.
         """
-        response = response.strip() if response else ""
         prompt = prompt.strip() if prompt else ""
+        response = response.strip() if response else ""
 
-        if not response:
-            raise ValueError("response is required for coherence evaluation")
+        if not prompt or not response:
+            raise ValueError(
+                "prompt and response are required for coherence evaluation"
+            )
 
         # Build inputs dictionary
         inputs = {"response": response, "prompt": prompt}
 
         payload = {
             "evaluator_name": self.name,
-            "parameters": {},
+            "parameters": {
+                "model": self.model,
+                "credential": self.credential,
+            },
             "inputs": inputs,
         }
 

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/fiddler_evals/evaluators/conciseness.py

@@ -1,8 +1,8 @@
-from fiddler_evals.evaluators.base import FiddlerEvaluator
+from fiddler_evals.evaluators.base import FiddlerLLMAAJEvaluator
 from fiddler_evals.pydantic_models.score import Score
 
 
-class Conciseness(FiddlerEvaluator):
+class Conciseness(FiddlerLLMAAJEvaluator):
     """Evaluator to assess how concise and to-the-point an answer is.
 
     The Conciseness evaluator measures whether an LLM's answer is appropriately
@@ -77,7 +77,10 @@ class Conciseness(FiddlerEvaluator):
 
         payload = {
             "evaluator_name": self.name,
-            "parameters": {},
+            "parameters": {
+                "model": self.model,
+                "credential": self.credential,
+            },
             "inputs": {"response": response},
         }
 

{fiddler_evals-0.1.1.dev14 → fiddler_evals-0.2.0rc1}/fiddler_evals/evaluators/tests/test_answer_relevance.py

@@ -6,15 +6,20 @@ import responses
 from fiddler_evals.constants import CONTENT_TYPE_HEADER_KEY, JSON_CONTENT_TYPE
 from fiddler_evals.evaluators.answer_relevance import AnswerRelevance
 from fiddler_evals.pydantic_models.score import Score, ScoreStatus
-from fiddler_evals.tests.constants import URL
+from fiddler_evals.tests.constants import LLM_GATEWAY_CREDENTIAL, LLM_GATEWAY_MODEL, URL
+
+
+@pytest.fixture()
+def evaluator() -> AnswerRelevance:
+    """Create an AnswerRelevance evaluator."""
+    return AnswerRelevance(model=LLM_GATEWAY_MODEL, credential=LLM_GATEWAY_CREDENTIAL)
 
 
 @responses.activate
-def test_answer_relevance_relevant_answer() -> None:
+def test_answer_relevance_relevant_answer(evaluator: AnswerRelevance) -> None:
     """When evaluating a relevant answer
     Then it should return score 1.0
     And should include proper reasoning."""
-    evaluator = AnswerRelevance()
 
     # Mock the API response
     mock_response = {
@@ -63,17 +68,19 @@ def test_answer_relevance_relevant_answer() -> None:
     # Verify request body
     request_body = json.loads(request.body)
     assert request_body["evaluator_name"] == "answer_relevance"
-    assert request_body["parameters"] == {}
+    assert request_body["parameters"] == {
+        "credential": LLM_GATEWAY_CREDENTIAL,
+        "model": LLM_GATEWAY_MODEL,
+    }
     assert request_body["inputs"]["prompt"] == "What is the capital of France?"
     assert request_body["inputs"]["response"] == "The capital of France is Paris."
 
 
 @responses.activate
-def test_answer_relevance_irrelevant_answer() -> None:
+def test_answer_relevance_irrelevant_answer(evaluator: AnswerRelevance) -> None:
     """When evaluating an irrelevant answer
     Then it should return score 0.0
     And should include proper reasoning."""
-    evaluator = AnswerRelevance()
 
     # Mock the API response
     mock_response = {
@@ -122,17 +129,19 @@ def test_answer_relevance_irrelevant_answer() -> None:
     # Verify request body
     request_body = json.loads(request.body)
     assert request_body["evaluator_name"] == "answer_relevance"
-    assert request_body["parameters"] == {}
+    assert request_body["parameters"] == {
+        "credential": LLM_GATEWAY_CREDENTIAL,
+        "model": LLM_GATEWAY_MODEL,
+    }
     assert request_body["inputs"]["prompt"] == "What is the capital of France?"
     assert request_body["inputs"]["response"] == "I like pizza and Italian food."
 
 
 @responses.activate
-def test_answer_relevance_missing_reasoning() -> None:
+def test_answer_relevance_missing_reasoning(evaluator: AnswerRelevance) -> None:
     """When API response has no reasoning
     Then it should return score with None reasoning
     And should handle missing fields gracefully."""
-    evaluator = AnswerRelevance()
 
     # Mock the API response without reasoning
     mock_response = {
@@ -173,17 +182,19 @@ def test_answer_relevance_missing_reasoning() -> None:
     # Verify request body
     request_body = json.loads(request.body)
     assert request_body["evaluator_name"] == "answer_relevance"
-    assert request_body["parameters"] == {}
+    assert request_body["parameters"] == {
+        "credential": LLM_GATEWAY_CREDENTIAL,
+        "model": LLM_GATEWAY_MODEL,
+    }
     assert request_body["inputs"]["prompt"] == "What is the capital of France?"
     assert request_body["inputs"]["response"] == "The capital of France is Paris."
 
 
 @responses.activate
-def test_answer_relevance_api_error_handling() -> None:
+def test_answer_relevance_api_error_handling(evaluator: AnswerRelevance) -> None:
     """When API call raises an exception
     Then it should propagate the exception
     And should not return a score."""
-    evaluator = AnswerRelevance()
 
     # Mock API error response
     responses.post(
@@ -206,7 +217,10 @@ def test_answer_relevance_api_error_handling() -> None:
     # Verify request body
     request_body = json.loads(request.body)
     assert request_body["evaluator_name"] == "answer_relevance"
-    assert request_body["parameters"] == {}
+    assert request_body["parameters"] == {
+        "credential": LLM_GATEWAY_CREDENTIAL,
+        "model": LLM_GATEWAY_MODEL,
+    }
     assert request_body["inputs"]["prompt"] == "What is the capital of France?"
     assert request_body["inputs"]["response"] == "The capital of France is Paris."
 
@@ -224,11 +238,12 @@ def test_answer_relevance_api_error_handling() -> None:
         ("What is the capital of France?", "   \t\n  "),
     ],
 )
-def test_answer_relevance_validation_errors(prompt, response) -> None:
+def test_answer_relevance_validation_errors(
+    evaluator: AnswerRelevance, prompt: str, response: str
+) -> None:
     """When providing invalid prompt or response
     Then it should raise appropriate ValueError
     And should not make API call."""
-    evaluator = AnswerRelevance()
 
     with pytest.raises(ValueError, match="prompt and response are required"):
         evaluator.score(prompt=prompt, response=response)