validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/prompt_validation/Robustness.py

@@ -2,33 +2,72 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from typing import List
-
 import pandas as pd
 
+from validmind import tags, tasks
 from validmind.errors import MissingRequiredTestInputError, SkipTestError
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
 
 from .ai_powered_test import call_model, missing_prompt_message
 
+SYSTEM = """
+You are a prompt evaluation researcher AI who is tasked with testing the robustness of LLM prompts.
+
+Consider the following guidelines:
+'''
+LLM prompts are used to guide a model to generate specific outputs or solve specific tasks.
+These prompts can be more or less robust, meaning that they can be more or less susceptible to breaking especially when the output needs to be a specific type.
+A robust prompt ensures consistent performance and reduces the likelihood of unexpected or off-tangent outputs.
+This consistency is vital for applications where predictability and reliability of the LLM's response are paramount.
+'''
+
+Consider the user-submitted prompt template and generate an input for the variable in the template (denoted by brackets) that tests the robustness of the prompt.
+Contradictions, edge cases, typos, bad phrasing, distracting, complex or out-of-place words and phrases are just some of the strategies you can use when generating inputs.
+Be creative and think step-by-step how you would break the prompt.
+Then generate {num_tests} inputs for the user-submitted prompt template that would break the prompt.
+Each input should be different from the others.
+Each input should be retured as a new line in your response.
+Respond only with the values to be inserted into the prompt template and do not include quotes, explanations or any extra text.
+
+Example:
+
+User-provided prompt:
+```
+Analyse the following sentence and output its sentiment:
+\\{sentence\\}
+```
+
+Your response (generated inputs):
+```
+I am a happy cat
+You are a bad person
+My name is bob
+James' friend is really sick
+```
+"""
 
-@dataclass
-class Robustness(ThresholdTest):
+USER = """
+Prompt:
+```
+{prompt_to_test}
+```
+Input:
+"""
+
+
+@tags("llm", "zero_shot", "few_shot")
+@tasks("text_classification", "text_summarization")
+def Robustness(model, dataset, num_tests=10):
     """
-    Assesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts.
+    Assesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts. This test
+    specifically measures the model's ability to generate correct classifications with the given prompt even when the
+    inputs are edge cases or otherwise difficult to classify.
 
     ### Purpose
 
     The Robustness test is meant to evaluate the resilience and reliability of prompts provided to a Language Learning
     Model (LLM). The aim of this test is to guarantee that the prompts consistently generate accurate and expected
-    outputs, even in diverse or challenging scenarios.
+    outputs, even in diverse or challenging scenarios. This test is only applicable to LLM-powered text classification
+    tasks where the prompt has a single input variable.
 
     ### Test Mechanism
 
@@ -58,114 +97,34 @@ class Robustness(ThresholdTest):
     - The test may not account for all potential conditions or alterations that could show up in practical use
     scenarios.
     """
+    if not hasattr(model, "prompt"):
+        raise MissingRequiredTestInputError(missing_prompt_message)
 
-    name = "robustness"
-    required_inputs = ["model"]
-    default_params = {"num_tests": 10}
-    tasks = ["text_classification", "text_summarization"]
-    tags = ["llm", "zero_shot", "few_shot"]
-
-    system_prompt = '''
-You are a prompt evaluation researcher AI who is tasked with testing the robustness of LLM prompts.
+    if not model.prompt.variables or len(model.prompt.variables) > 1:
+        raise SkipTestError("Robustness only supports single-variable prompts for now")
 
-Consider the following guidelines:
-```
-LLM prompts are used to guide a model to generate specific outputs or solve specific tasks. These prompts can be more or less robust, meaning that they can be more or less susceptible to breaking especially when the output needs to be a specific type.
-A robust prompt ensures consistent performance and reduces the likelihood of unexpected or off-tangent outputs. This consistency is vital for applications where predictability and reliability of the LLM's response are paramount.
-```
-
-Consider the user-submitted prompt template and generate an input for the variable in the template (denoted by brackets) that tests the robustness of the prompt.
-Contradictions, edge cases, typos, bad phrasing, distracting, complex or out-of-place words and phrases are just some of the strategies you can use when generating inputs.
-Be creative and think step-by-step how you would break the prompt. Then generate an input for the user-submitted prompt template that would break the prompt.
-Respond only with the value to be inserted into the prompt template and do not include quotes, explanations or any extra text.
-Example:
-Prompt:
-"""
-Analyse the following sentence and output its sentiment\n{sentence}
-"""
-Input:
-Nonsense string that has no sentiment
-'''.strip()
-    user_prompt = '''
-Prompt:
-"""
-{prompt_to_test}
-"""
-Input:
-'''.strip()
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        results_table = [
-            {
-                "Test Input": result.values["input"],
-                "Model Output": result.values["output"],
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-            for result in results
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pd.DataFrame(results_table),
-                    metadata=ResultTableMetadata(
-                        title="Robustness Test on Prompt",
-                    ),
-                )
-            ]
+    target_class_labels = dataset.target_classes()
+    if len(target_class_labels) > 10:
+        raise SkipTestError(
+            "Too many target classes to test robustness. Skipping test."
         )
 
-    def run(self):
-        if not hasattr(self.inputs.model, "prompt"):
-            raise MissingRequiredTestInputError(missing_prompt_message)
-
-        # TODO: add support for multi-variable prompts
-        if (
-            not self.inputs.model.prompt.variables
-            or len(self.inputs.model.prompt.variables) > 1
-        ):
-            raise SkipTestError(
-                "Robustness only supports single-variable prompts for now"
-            )
-
-        target_class_labels = self.inputs.dataset.target_classes()
-        # Guard against too many classes (maybe not a classification model)
-        if len(target_class_labels) > 10:
-            raise SkipTestError(
-                "Too many target classes to test robustness. Skipping test."
-            )
-
-        results = []
-
-        for _ in range(self.params["num_tests"]):
-            response = call_model(
-                system_prompt=self.system_prompt,
-                user_prompt=self.user_prompt.format(
-                    variables="\n".join(self.inputs.model.prompt.variables),
-                    prompt_to_test=self.inputs.model.prompt.template,
-                ),
-            )
-
-            test_input_df = pd.DataFrame(
-                [response],
-                columns=self.inputs.model.prompt.variables,
-            )
-            result = self.inputs.model.predict(test_input_df)[0]
-
-            fail = False
-            if result not in target_class_labels:
-                fail = True
-
-            results.append(
-                ThresholdTestResult(
-                    passed=not fail,
-                    values={
-                        "input": response,
-                        "output": result,
-                    },
-                )
-            )
-
-        return self.cache_results(
-            results, passed=all([result.passed for result in results])
-        )
+    generated_inputs = call_model(
+        system_prompt=SYSTEM.format(num_tests=num_tests),
+        user_prompt=USER.format(prompt_to_test=model.prompt.template),
+    ).split("\n")
+
+    responses = model.predict(
+        pd.DataFrame(generated_inputs, columns=model.prompt.variables)
+    )
+
+    results = [
+        {
+            "Generated Input": generated_input,
+            "Model Response": response,
+            "Pass/Fail": "Pass" if response in target_class_labels else "Fail",
+        }
+        for generated_input, response in zip(generated_inputs, responses)
+    ]
+
+    return results, all(result["Pass/Fail"] == "Pass" for result in results)

validmind/tests/prompt_validation/Specificity.py

@@ -2,19 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from typing import List
-
-import pandas as pd
-
+from validmind import tags, tasks
 from validmind.errors import MissingRequiredTestInputError
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
 
 from .ai_powered_test import (
     call_model,
@@ -23,9 +12,47 @@ from .ai_powered_test import (
     missing_prompt_message,
 )
 
+SYSTEM = """
+You are a prompt evaluation AI.
+You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics.
+You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
+
+Consider the following documentation regarding specificity in prompts and utilize it to grade the user-submitted prompt:
+```
+Prompts that are detailed and descriptive often yield better and more accurate results from an LLM.
+Rather than relying on specific keywords or tokens, it's crucial to have a well-structured and descriptive prompt.
+Including relevant examples within the prompt can be particularly effective, guiding the LLM to produce outputs in desired formats.
+However, it's essential to strike a balance. While prompts need to be detailed, they shouldn't be overloaded with unnecessary information.
+The emphasis should always be on relevancy and conciseness, considering there are limitations to how long a prompt can be.
+
+Example:
+Imagine wanting an LLM to extract specific details from a given text.
+A vague prompt might yield varied results.
+However, with a prompt like, "Extract the names of all characters and the cities they visited from the text", the LLM is guided more precisely towards the desired information extraction.
+```
+
+Score the specificity of the user-submitted prompt.
+Return a score from 1 to 10 where 10 is a perfect score.
+Also provide a short explanation for your score.
+
+Response Format:
+```
+Score: <score>
+Explanation: <explanation>
+```
+"""
+
+USER = """
+Prompt:
+'''
+{prompt_to_test}
+'''
+"""
 
-@dataclass
-class Specificity(ThresholdTest):
+
+@tags("llm", "zero_shot", "few_shot")
+@tasks("text_classification", "text_summarization")
+def Specificity(model, min_threshold=7):
     """
     Evaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity, detail,
     and relevance.
@@ -64,84 +91,23 @@ class Specificity(ThresholdTest):
     - Striking a balance between specificity and verbosity can be challenging, as overly detailed prompts might confuse
     or mislead the model
     """
-
-    name = "specificity"
-    required_inputs = ["model.prompt"]
-    default_params = {"min_threshold": 7}
-    tasks = ["text_classification", "text_summarization"]
-    tags = ["llm", "zero_shot", "few_shot"]
-
-    system_prompt = """
-You are a prompt evaluation AI. You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics. You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
-
-Consider the following documentation regarding specificity in prompts and utilize it to grade the user-submitted prompt:
-```
-Prompts that are detailed and descriptive often yield better and more accurate results from an LLM. Rather than relying on specific keywords or tokens, it's crucial to have a well-structured and descriptive prompt. Including relevant examples within the prompt can be particularly effective, guiding the LLM to produce outputs in desired formats. However, it's essential to strike a balance. While prompts need to be detailed, they shouldn't be overloaded with unnecessary information. The emphasis should always be on relevancy and conciseness, considering there are limitations to how long a prompt can be.
-
-Example:
-Imagine wanting an LLM to extract specific details from a given text. A vague prompt might yield varied results. However, with a prompt like, "Extract the names of all characters and the cities they visited from the text", the LLM is guided more precisely towards the desired information extraction.
-```
-
-Score the specificity of the user-submitted prompt. Return a score from 1 to 10 where 10 is a perfect score. Also provide a short explanation for your score
-
-Response Format:
-```
-Score: <score>
-Explanation: <explanation>
-```
-""".strip()
-    user_prompt = '''
-Prompt:
-"""
-{prompt_to_test}
-"""
-'''.strip()
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        result = results[0]
-        results_table = [
-            {
-                "Score": result.values["score"],
-                "Threshold": result.values["threshold"],
-                "Explanation": result.values["explanation"],
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pd.DataFrame(results_table),
-                    metadata=ResultTableMetadata(
-                        title="Specificity Test for LLM Prompt",
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        if not hasattr(self.inputs.model, "prompt"):
-            raise MissingRequiredTestInputError(missing_prompt_message)
-
-        response = call_model(
-            system_prompt=self.system_prompt,
-            user_prompt=self.user_prompt.format(
-                prompt_to_test=self.inputs.model.prompt.template
-            ),
-        )
-        score = get_score(response)
-        explanation = get_explanation(response)
-
-        passed = score > self.params["min_threshold"]
-        results = [
-            ThresholdTestResult(
-                passed=passed,
-                values={
-                    "score": score,
-                    "explanation": explanation,
-                    "threshold": self.params["min_threshold"],
-                },
-            )
-        ]
-
-        return self.cache_results(results, passed=passed)
+    if not hasattr(model, "prompt"):
+        raise MissingRequiredTestInputError(missing_prompt_message)
+
+    response = call_model(
+        system_prompt=SYSTEM,
+        user_prompt=USER.format(prompt_to_test=model.prompt.template),
+    )
+    score = get_score(response)
+    explanation = get_explanation(response)
+
+    passed = score > min_threshold
+
+    return [
+        {
+            "Score": score,
+            "Threshold": min_threshold,
+            "Explanation": explanation,
+            "Pass/Fail": "Pass" if passed else "Fail",
+        }
+    ], passed

validmind/tests/prompt_validation/ai_powered_test.py

@@ -36,8 +36,8 @@ def call_model(
         client.chat.completions.create(
             model=model,
             messages=[
-                {"role": "system", "content": system_prompt},
-                {"role": "user", "content": user_prompt},
+                {"role": "system", "content": system_prompt.strip("\n").strip()},
+                {"role": "user", "content": user_prompt.strip("\n").strip()},
             ],
             temperature=temperature,
             seed=seed,