fiddler-evals 0.1.1.dev13__tar.gz → 0.2.0__tar.gz

{fiddler_evals-0.1.1.dev13/fiddler_evals.egg-info → fiddler_evals-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: fiddler-evals
-Version: 0.1.1.dev13
+Version: 0.2.0
 Summary: Python SDK for evaluating LLM Applications
 Author-email: Fiddler AI <support@fiddler.ai>
 Maintainer-email: Fiddler AI <support@fiddler.ai>
@@ -15,7 +15,6 @@ Requires-Dist: requests<3
 Requires-Dist: pydantic>=2.0.0
 Requires-Dist: tqdm
 Requires-Dist: typing-extensions<5,>=4.6.0
-Requires-Dist: pandas>=1.2.5
 Requires-Dist: python-decouple
 Provides-Extra: pandas
 Requires-Dist: pandas>=1.2.5; extra == "pandas"
@@ -60,7 +59,7 @@ pip install --upgrade --pre fiddler-evals
 from fiddler_evals import init
 
 # Initialize connection
-init(url='https://your-org.fiddler.ai', token='your-api-token')
+init(url='https://your-instance.fiddler.ai', token='your-api-token')
 ```
 
 ### 2. Create Project Structure
@@ -101,19 +100,32 @@ dataset.insert(test_cases)
 
 ### 4. Use Built-in Evaluators
 
+**Configure LLM Gateway provider:**
+
+Add an LLM provider via UI (**Settings > LLM Gateway**) to use Fiddler's pre-build LLM-as-a-Judge evaluators.
+LLM-as-a-Judge evaluators require a `model` parameter in the format `{provider}/{model}` (e.g., `openai/gpt-4o`) and an optional `credential` parameter for LLM Gateway authentication.
+
 ```python
 from fiddler_evals.evaluators import (
     AnswerRelevance, Coherence, Conciseness,
-    Toxicity, Sentiment, RegexSearch
+    Sentiment, RegexSearch
 )
 
-# Test individual evaluators
-relevance_evaluator = AnswerRelevance()
+# Test LLM-as-a-Judge evaluators (require model parameter)
+relevance_evaluator = AnswerRelevance(
+    model="openai/gpt-4o",           # Required: LLM Gateway model in {provider}/{model} format
+    credential="my-openai-cred"      # Optional: LLM Gateway credential name
+)
 score = relevance_evaluator.score(
     prompt="What is the capital of France?",
     response="Paris is the capital of France."
 )
 print(f"Score: {score.value} - {score.reasoning}")
+
+# Test other evaluators (no model parameter needed)
+sentiment_evaluator = Sentiment()
+scores = sentiment_evaluator.score(text="This is a helpful response.")
+print("Sentiments:", [f'{score.name}: {score.value}' for score in scores])
 ```
 
 ### 5. Create Custom Evaluators
@@ -128,8 +140,11 @@ class PolitenessEvaluator(Evaluator):
     Useful for customer service or chatbot applications.
     """
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, score_name_prefix: str = None, score_fn_kwargs_mapping: dict = None):
+        super().__init__(
+            score_name_prefix=score_name_prefix,
+            score_fn_kwargs_mapping=score_fn_kwargs_mapping
+        )
         self.polite_words = [
             'please', 'thank you', 'thanks', 'sorry', 'apologize',
             'appreciate', 'welcome', 'help', 'assist', 'glad'
@@ -151,13 +166,13 @@ class PolitenessEvaluator(Evaluator):
             reasoning = "No polite language detected"
 
         return Score(
-            name="politeness",
+            name=f"{self.score_name_prefix}politeness",
             evaluator_name=self.name,
             value=score_value,
             reasoning=reasoning
         )
 
-# Test the evaluator
+# Test the evaluator with different configurations
 politeness_evaluator = PolitenessEvaluator()
 
 polite_response = "Thank you for your question! I'd be happy to help you with that."
@@ -165,6 +180,17 @@ impolite_response = "I don't know. Figure it out yourself."
 
 print(f"Polite response score: {politeness_evaluator.score(polite_response).value}")
 print(f"Impolite response score: {politeness_evaluator.score(impolite_response).value}")
+
+# Use with different configurations
+customer_service_evaluator = PolitenessEvaluator(
+    score_name_prefix="customer_service",
+    score_fn_kwargs_mapping={"output": "response"}
+)
+
+support_evaluator = PolitenessEvaluator(
+    score_name_prefix="support",
+    score_fn_kwargs_mapping={"output": "answer"}
+)
 ```
 
 ### 5.1. Function-Based Evaluators
@@ -185,8 +211,8 @@ def contains_number_evaluator(output: str) -> float:
 
 # Use functions directly in evaluators list
 evaluators = [
-    AnswerRelevance(),
-    Conciseness(),
+    AnswerRelevance(model="openai/gpt-4o", credential="my-openai-cred"),
+    Conciseness(model="openai/gpt-4o", credential="my-openai-cred"),
     word_count_evaluator,        # Function evaluator
     contains_number_evaluator,   # Function evaluator
 ]
@@ -215,12 +241,42 @@ def my_llm_task(inputs: dict, extras: dict, metadata: dict) -> dict:
     answer = call_your_llm(question)
     return {"answer": answer}
 
-# Set up evaluators
+# Set up evaluators with different configurations
 evaluators = [
-    AnswerRelevance(),
-    Conciseness(),
-    Sentiment(),
-    PolitenessEvaluator(),
+    # LLM-as-a-Judge evaluators (require model parameter)
+    AnswerRelevance(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+    Conciseness(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+
+    # Other evaluators
+    Sentiment(score_name_prefix="primary"),
+
+    # Custom evaluators with specific mappings
+    PolitenessEvaluator(
+        score_name_prefix="quality",
+        score_fn_kwargs_mapping={"output": "answer"}
+    ),
+
+    # Multiple instances of same evaluator for different fields
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="question",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "question"}
+    ),
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="answer",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "answer"}
+    ),
 ]
 
 # Run evaluation
@@ -231,9 +287,8 @@ experiment_result = evaluate(
     name_prefix="my_evaluation",
     description="Comprehensive LLM evaluation",
     score_fn_kwargs_mapping={
-        "question": "question",
+        "question": lambda x: x["inputs"]["question"],
         "response": "answer",
-        "output": "answer",
         "text": "answer",
         "prompt": lambda x: x["inputs"]["question"],
     }
@@ -241,20 +296,25 @@ experiment_result = evaluate(
 
 print(f"Evaluated {len(experiment_result.results)} test cases")
 print(f"Generated {sum(len(result.scores) for result in experiment_result.results)} scores")
+
+# Results in organized score names:
+# "primary_answer_relevance", "primary_conciseness", "primary_sentiment",
+# "quality_politeness", "question_has_number", "answer_has_number"
 ```
 
 ## Built-in Evaluators
 
-| Evaluator | Purpose | Key Parameters |
-|-----------|---------|----------------|
-| `AnswerRelevance` | Checks if response addresses the question | `prompt`, `response` |
-| `Coherence` | Evaluates logical flow and consistency | `response`, `prompt` |
-| `Conciseness` | Measures response brevity and clarity | `response` |
-| `Toxicity` | Detects harmful or toxic content | `text` |
-| `Sentiment` | Analyzes emotional tone | `text` |
-| `RegexSearch` | Pattern matching for specific formats | `output`, `pattern` |
-| `FTLPromptSafety` | Compute safety scores for prompts | `text` |
-| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | `response`, `context` |
+| Evaluator | Purpose | Constructor Parameters | Score Parameters |
+|-----------|---------|------------------------|------------------|
+| `AnswerRelevance` | Checks if response addresses the question | `model` (required), `credential` (required) | `prompt`, `response` |
+| `Coherence` | Evaluates logical flow and consistency | `model` (required), `credential` (required) | `response`, `prompt` (optional) |
+| `Conciseness` | Measures response brevity and clarity | `model` (required), `credential` (required) | `response` |
+| `Sentiment` | Analyzes emotional tone | - | `text` |
+| `RegexSearch` | Pattern matching for specific formats | `pattern` (required) | `output` |
+| `FTLPromptSafety` | Compute safety scores for prompts | - | `text` |
+| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | - | `response`, `context` |
+
+**Note:** Evaluators marked with `model` and `credential` parameters are LLM-as-a-Judge evaluators that require an LLM Gateway model. The `model` parameter should be in `{provider}/{model}` format (e.g., `openai/gpt-4o`). The `credential` parameter is the name of the LLM Gateway credential for authentication.
 
 ## Data Import Options
 
@@ -326,6 +386,79 @@ score_fn_kwargs_mapping={
 }
 ```
 
+### Multiple Evaluator Instances with Different Mappings
+
+You can create multiple instances of the same evaluator with different parameter mappings and score name prefixes to evaluate different aspects of your outputs. Use `score_name_prefix` to organize and distinguish scores when using multiple evaluator instances:
+
+```python
+from fiddler_evals.evaluators import RegexSearch
+
+# Create multiple RegexSearch evaluators for different fields
+evaluators = [
+    # Check for numbers in the question
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="question",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "question"}
+    ),
+    # Check for numbers in the answer
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="answer",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "answer"}
+    ),
+    # Check for capital letters in the answer
+    RegexSearch(
+        pattern=r"[A-Z]",
+        score_name_prefix="answer",
+        score_name="has_caps",
+        score_fn_kwargs_mapping={"output": "answer"}
+    )
+]
+
+# Run evaluation
+experiment_result = evaluate(
+    dataset=dataset,
+    task=my_llm_task,
+    evaluators=evaluators,
+    score_fn_kwargs_mapping={
+        "question": lambda x: x["inputs"]["question"]
+    }
+)
+
+# Results in scores named:
+# "question_has_number", "answer_has_number", "answer_has_caps"
+```
+
+### Parameter Mapping Priority
+
+When both evaluator-level and evaluation-level mappings are present, evaluator-level mappings take precedence:
+
+```python
+# Evaluator-level mapping (higher priority)
+evaluator = RegexSearch(
+    pattern=r"\d+",
+    score_fn_kwargs_mapping={"output": "answer"}  # This takes precedence
+)
+
+# Evaluation-level mapping (lower priority)
+experiment_result = evaluate(
+    dataset=dataset,
+    task=my_llm_task,
+    evaluators=[evaluator],
+    score_fn_kwargs_mapping={
+        "output": "question"  # This is ignored due to evaluator-level mapping
+    }
+)
+```
+
+**Mapping Priority (highest to lowest):**
+1. Evaluator-level `score_fn_kwargs_mapping` (set in evaluator constructor)
+2. Evaluation-level `score_fn_kwargs_mapping` (passed to evaluate function)
+3. Default parameter resolution
+
 ### Experiment Metadata
 ```python
 experiment_result = evaluate(

{fiddler_evals-0.1.1.dev13 → fiddler_evals-0.2.0}/PUBLIC.md

@@ -38,7 +38,7 @@ pip install --upgrade --pre fiddler-evals
 from fiddler_evals import init
 
 # Initialize connection
-init(url='https://your-org.fiddler.ai', token='your-api-token')
+init(url='https://your-instance.fiddler.ai', token='your-api-token')
 ```
 
 ### 2. Create Project Structure
@@ -79,19 +79,32 @@ dataset.insert(test_cases)
 
 ### 4. Use Built-in Evaluators
 
+**Configure LLM Gateway provider:**
+
+Add an LLM provider via UI (**Settings > LLM Gateway**) to use Fiddler's pre-build LLM-as-a-Judge evaluators.
+LLM-as-a-Judge evaluators require a `model` parameter in the format `{provider}/{model}` (e.g., `openai/gpt-4o`) and an optional `credential` parameter for LLM Gateway authentication.
+
 ```python
 from fiddler_evals.evaluators import (
     AnswerRelevance, Coherence, Conciseness,
-    Toxicity, Sentiment, RegexSearch
+    Sentiment, RegexSearch
 )
 
-# Test individual evaluators
-relevance_evaluator = AnswerRelevance()
+# Test LLM-as-a-Judge evaluators (require model parameter)
+relevance_evaluator = AnswerRelevance(
+    model="openai/gpt-4o",           # Required: LLM Gateway model in {provider}/{model} format
+    credential="my-openai-cred"      # Optional: LLM Gateway credential name
+)
 score = relevance_evaluator.score(
     prompt="What is the capital of France?",
     response="Paris is the capital of France."
 )
 print(f"Score: {score.value} - {score.reasoning}")
+
+# Test other evaluators (no model parameter needed)
+sentiment_evaluator = Sentiment()
+scores = sentiment_evaluator.score(text="This is a helpful response.")
+print("Sentiments:", [f'{score.name}: {score.value}' for score in scores])
 ```
 
 ### 5. Create Custom Evaluators
@@ -106,8 +119,11 @@ class PolitenessEvaluator(Evaluator):
     Useful for customer service or chatbot applications.
     """
 
-    def __init__(self):
-        super().__init__()
+    def __init__(self, score_name_prefix: str = None, score_fn_kwargs_mapping: dict = None):
+        super().__init__(
+            score_name_prefix=score_name_prefix,
+            score_fn_kwargs_mapping=score_fn_kwargs_mapping
+        )
         self.polite_words = [
             'please', 'thank you', 'thanks', 'sorry', 'apologize',
             'appreciate', 'welcome', 'help', 'assist', 'glad'
@@ -129,13 +145,13 @@ class PolitenessEvaluator(Evaluator):
             reasoning = "No polite language detected"
 
         return Score(
-            name="politeness",
+            name=f"{self.score_name_prefix}politeness",
             evaluator_name=self.name,
             value=score_value,
             reasoning=reasoning
         )
 
-# Test the evaluator
+# Test the evaluator with different configurations
 politeness_evaluator = PolitenessEvaluator()
 
 polite_response = "Thank you for your question! I'd be happy to help you with that."
@@ -143,6 +159,17 @@ impolite_response = "I don't know. Figure it out yourself."
 
 print(f"Polite response score: {politeness_evaluator.score(polite_response).value}")
 print(f"Impolite response score: {politeness_evaluator.score(impolite_response).value}")
+
+# Use with different configurations
+customer_service_evaluator = PolitenessEvaluator(
+    score_name_prefix="customer_service",
+    score_fn_kwargs_mapping={"output": "response"}
+)
+
+support_evaluator = PolitenessEvaluator(
+    score_name_prefix="support",
+    score_fn_kwargs_mapping={"output": "answer"}
+)
 ```
 
 ### 5.1. Function-Based Evaluators
@@ -163,8 +190,8 @@ def contains_number_evaluator(output: str) -> float:
 
 # Use functions directly in evaluators list
 evaluators = [
-    AnswerRelevance(),
-    Conciseness(),
+    AnswerRelevance(model="openai/gpt-4o", credential="my-openai-cred"),
+    Conciseness(model="openai/gpt-4o", credential="my-openai-cred"),
     word_count_evaluator,        # Function evaluator
     contains_number_evaluator,   # Function evaluator
 ]
@@ -193,12 +220,42 @@ def my_llm_task(inputs: dict, extras: dict, metadata: dict) -> dict:
     answer = call_your_llm(question)
     return {"answer": answer}
 
-# Set up evaluators
+# Set up evaluators with different configurations
 evaluators = [
-    AnswerRelevance(),
-    Conciseness(),
-    Sentiment(),
-    PolitenessEvaluator(),
+    # LLM-as-a-Judge evaluators (require model parameter)
+    AnswerRelevance(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+    Conciseness(
+        model="openai/gpt-4o",
+        credential="my-openai-cred",
+        score_name_prefix="primary"
+    ),
+
+    # Other evaluators
+    Sentiment(score_name_prefix="primary"),
+
+    # Custom evaluators with specific mappings
+    PolitenessEvaluator(
+        score_name_prefix="quality",
+        score_fn_kwargs_mapping={"output": "answer"}
+    ),
+
+    # Multiple instances of same evaluator for different fields
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="question",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "question"}
+    ),
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="answer",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "answer"}
+    ),
 ]
 
 # Run evaluation
@@ -209,9 +266,8 @@ experiment_result = evaluate(
     name_prefix="my_evaluation",
     description="Comprehensive LLM evaluation",
     score_fn_kwargs_mapping={
-        "question": "question",
+        "question": lambda x: x["inputs"]["question"],
         "response": "answer",
-        "output": "answer",
         "text": "answer",
         "prompt": lambda x: x["inputs"]["question"],
     }
@@ -219,20 +275,25 @@ experiment_result = evaluate(
 
 print(f"Evaluated {len(experiment_result.results)} test cases")
 print(f"Generated {sum(len(result.scores) for result in experiment_result.results)} scores")
+
+# Results in organized score names:
+# "primary_answer_relevance", "primary_conciseness", "primary_sentiment",
+# "quality_politeness", "question_has_number", "answer_has_number"
 ```
 
 ## Built-in Evaluators
 
-| Evaluator | Purpose | Key Parameters |
-|-----------|---------|----------------|
-| `AnswerRelevance` | Checks if response addresses the question | `prompt`, `response` |
-| `Coherence` | Evaluates logical flow and consistency | `response`, `prompt` |
-| `Conciseness` | Measures response brevity and clarity | `response` |
-| `Toxicity` | Detects harmful or toxic content | `text` |
-| `Sentiment` | Analyzes emotional tone | `text` |
-| `RegexSearch` | Pattern matching for specific formats | `output`, `pattern` |
-| `FTLPromptSafety` | Compute safety scores for prompts | `text` |
-| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | `response`, `context` |
+| Evaluator | Purpose | Constructor Parameters | Score Parameters |
+|-----------|---------|------------------------|------------------|
+| `AnswerRelevance` | Checks if response addresses the question | `model` (required), `credential` (required) | `prompt`, `response` |
+| `Coherence` | Evaluates logical flow and consistency | `model` (required), `credential` (required) | `response`, `prompt` (optional) |
+| `Conciseness` | Measures response brevity and clarity | `model` (required), `credential` (required) | `response` |
+| `Sentiment` | Analyzes emotional tone | - | `text` |
+| `RegexSearch` | Pattern matching for specific formats | `pattern` (required) | `output` |
+| `FTLPromptSafety` | Compute safety scores for prompts | - | `text` |
+| `FTLResponseFaithfulness` | Evaluate faithfulness of LLM responses | - | `response`, `context` |
+
+**Note:** Evaluators marked with `model` and `credential` parameters are LLM-as-a-Judge evaluators that require an LLM Gateway model. The `model` parameter should be in `{provider}/{model}` format (e.g., `openai/gpt-4o`). The `credential` parameter is the name of the LLM Gateway credential for authentication.
 
 ## Data Import Options
 
@@ -304,6 +365,79 @@ score_fn_kwargs_mapping={
 }
 ```
 
+### Multiple Evaluator Instances with Different Mappings
+
+You can create multiple instances of the same evaluator with different parameter mappings and score name prefixes to evaluate different aspects of your outputs. Use `score_name_prefix` to organize and distinguish scores when using multiple evaluator instances:
+
+```python
+from fiddler_evals.evaluators import RegexSearch
+
+# Create multiple RegexSearch evaluators for different fields
+evaluators = [
+    # Check for numbers in the question
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="question",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "question"}
+    ),
+    # Check for numbers in the answer
+    RegexSearch(
+        pattern=r"\d+",
+        score_name_prefix="answer",
+        score_name="has_number",
+        score_fn_kwargs_mapping={"output": "answer"}
+    ),
+    # Check for capital letters in the answer
+    RegexSearch(
+        pattern=r"[A-Z]",
+        score_name_prefix="answer",
+        score_name="has_caps",
+        score_fn_kwargs_mapping={"output": "answer"}
+    )
+]
+
+# Run evaluation
+experiment_result = evaluate(
+    dataset=dataset,
+    task=my_llm_task,
+    evaluators=evaluators,
+    score_fn_kwargs_mapping={
+        "question": lambda x: x["inputs"]["question"]
+    }
+)
+
+# Results in scores named:
+# "question_has_number", "answer_has_number", "answer_has_caps"
+```
+
+### Parameter Mapping Priority
+
+When both evaluator-level and evaluation-level mappings are present, evaluator-level mappings take precedence:
+
+```python
+# Evaluator-level mapping (higher priority)
+evaluator = RegexSearch(
+    pattern=r"\d+",
+    score_fn_kwargs_mapping={"output": "answer"}  # This takes precedence
+)
+
+# Evaluation-level mapping (lower priority)
+experiment_result = evaluate(
+    dataset=dataset,
+    task=my_llm_task,
+    evaluators=[evaluator],
+    score_fn_kwargs_mapping={
+        "output": "question"  # This is ignored due to evaluator-level mapping
+    }
+)
+```
+
+**Mapping Priority (highest to lowest):**
+1. Evaluator-level `score_fn_kwargs_mapping` (set in evaluator constructor)
+2. Evaluation-level `score_fn_kwargs_mapping` (passed to evaluate function)
+3. Default parameter resolution
+
 ### Experiment Metadata
 ```python
 experiment_result = evaluate(

fiddler_evals-0.2.0/fiddler_evals/VERSION

@@ -0,0 +1 @@
+0.2.0

{fiddler_evals-0.1.1.dev13 → fiddler_evals-0.2.0}/fiddler_evals/__init__.py

@@ -20,7 +20,6 @@ from fiddler_evals.evaluators import (
     RegexSearch,
     Sentiment,
     TopicClassification,
-    Toxicity,
 )
 from fiddler_evals.evaluators.base import Evaluator
 from fiddler_evals.evaluators.eval_fn import EvalFn
@@ -55,7 +54,6 @@ __all__ = [
     "AnswerRelevance",
     "Coherence",
     "Conciseness",
-    "Toxicity",
     "Sentiment",
     "RegexSearch",
     "RegexMatch",