validmind 2.5.25__py3-none-any.whl → 2.6.8__py3-none-any.whl

validmind/tests/prompt_validation/Conciseness.py

@@ -2,19 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from typing import List
-
-import pandas as pd
-
+from validmind import tags, tasks
 from validmind.errors import MissingRequiredTestInputError
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
 
 from .ai_powered_test import (
     call_model,
@@ -23,9 +12,49 @@ from .ai_powered_test import (
     missing_prompt_message,
 )
 
+SYSTEM = """
+You are a prompt evaluation AI.
+You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics.
+You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
+
+Consider the following documentation regarding conciseness in prompts and utilize it to grade the user-submitted prompt:
+'''
+While detailed prompts can guide an LLM towards accurate results, excessive details can clutter the instruction and potentially lead to undesired outputs.
+Concise prompts are straightforward, reducing ambiguity and focusing the LLM's attention on the primary task.
+This is especially important considering there are limitations to the length of prompts that can be fed to an LLM.
+
+For an LLM tasked with summarizing a document, a verbose prompt might introduce unnecessary constraints or biases.
+A concise, effective prompt like:
+"Provide a brief summary highlighting the main points of the document"
+ensures that the LLM captures the essence of the content without being sidetracked.
+
+For example this prompt:
+"The description for this product should be fairly short, a few sentences only, and not too much more."
+could be better written like this:
+"Use a 3 to 5 sentence paragraph to describe this product."
+'''
+
+Score the user-submitted prompt on a scale of 1 to 10, with 10 being the best possible score.
+Provide an explanation for your score.
+
+Response Format:
+```
+Score: <score>
+Explanation: <explanation>
+```
+"""
 
-@dataclass
-class Conciseness(ThresholdTest):
+USER = """
+Prompt:
+```
+{prompt_to_test}
+```
+"""
+
+
+@tags("llm", "zero_shot", "few_shot")
+@tasks("text_classification", "text_summarization")
+def Conciseness(model, min_threshold=7):
     """
     Analyzes and grades the conciseness of prompts provided to a Large Language Model.
 
@@ -62,90 +91,23 @@ class Conciseness(ThresholdTest):
     - The predefined threshold for conciseness could be subjective and might need adjustment based on application.
     - The test is dependent on the LLM’s understanding of conciseness, which might vary from model to model.
     """
-
-    name = "conciseness"
-    required_inputs = ["model.prompt"]
-    default_params = {"min_threshold": 7}
-    tasks = ["text_classification", "text_summarization"]
-    tags = ["llm", "zero_shot", "few_shot"]
-
-    system_prompt = """
-You are a prompt evaluation AI. You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics. You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
-
-Consider the following documentation regarding conciseness in prompts and utilize it to grade the user-submitted prompt:
-'''
-While detailed prompts can guide an LLM towards accurate results, excessive details can clutter the instruction and potentially lead to undesired outputs. Concise prompts are straightforward, reducing ambiguity and focusing the LLM's attention on the primary task. This is especially important considering there are limitations to the length of prompts that can be fed to an LLM.
-
-For an LLM tasked with summarizing a document, a verbose prompt might introduce unnecessary constraints or biases. A concise, effective prompt like:
-"Provide a brief summary highlighting the main points of the document"
-ensures that the LLM captures the essence of the content without being sidetracked.
-
-For example this prompt:
-"The description for this product should be fairly short, a few sentences only, and not too much more."
-could be better written like this:
-"Use a 3 to 5 sentence paragraph to describe this product."
-'''
-
-Score the user-submitted prompt on a scale of 1 to 10, with 10 being the best possible score. Provide an explanation for your score.
-
-Response Format:
-```
-Score: <score>
-Explanation: <explanation>
-```
-""".strip()
-    user_prompt = '''
-Prompt:
-"""
-{prompt_to_test}
-"""
-'''.strip()
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        result = results[0]
-        results_table = [
-            {
-                "Score": result.values["score"],
-                "Threshold": result.values["threshold"],
-                "Explanation": result.values["explanation"],
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pd.DataFrame(results_table),
-                    metadata=ResultTableMetadata(
-                        title="Conciseness Test for LLM Prompt",
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        if not hasattr(self.inputs.model, "prompt"):
-            raise MissingRequiredTestInputError(missing_prompt_message)
-
-        response = call_model(
-            system_prompt=self.system_prompt,
-            user_prompt=self.user_prompt.format(
-                prompt_to_test=self.inputs.model.prompt.template
-            ),
-        )
-        score = get_score(response)
-        explanation = get_explanation(response)
-
-        passed = score > self.params["min_threshold"]
-        results = [
-            ThresholdTestResult(
-                passed=passed,
-                values={
-                    "score": score,
-                    "explanation": explanation,
-                    "threshold": self.params["min_threshold"],
-                },
-            )
-        ]
-
-        return self.cache_results(results, passed=passed)
+    if not hasattr(model, "prompt"):
+        raise MissingRequiredTestInputError(missing_prompt_message)
+
+    response = call_model(
+        system_prompt=SYSTEM,
+        user_prompt=USER.format(prompt_to_test=model.prompt.template),
+    )
+    score = get_score(response)
+    explanation = get_explanation(response)
+
+    passed = score > min_threshold
+
+    return [
+        {
+            "Score": score,
+            "Threshold": min_threshold,
+            "Explanation": explanation,
+            "Pass/Fail": "Pass" if passed else "Fail",
+        }
+    ], passed

validmind/tests/prompt_validation/Delimitation.py

@@ -2,19 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from typing import List
-
-import pandas as pd
-
+from validmind import tags, tasks
 from validmind.errors import MissingRequiredTestInputError
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
 
 from .ai_powered_test import (
     call_model,
@@ -23,9 +12,34 @@ from .ai_powered_test import (
     missing_prompt_message,
 )
 
+SYSTEM = """
+You are a prompt evaluation AI.
+You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics.
+You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
+
+LLM Prompts that include different sections and user inputs should be properly delimitated.
+Ideally, the prompt should use triple quotes or backticks or at least single quotes around any user input, reference text or code block etc.
+This is to ensure that the prompt is parsed correctly by the model, different pieces of the prompt are understood as separate and any user-provided inputs are not interpreted as part of the prompt.
+Identify any issues in the user-submitted prompt and give a score from 1 to 10, where 10 is a perfect score, based on the number and severity of issues.
+
+Response Format:
+```
+Score: <score>
+Explanation: <explanation>
+```
+"""
+
+USER = """
+Prompt:
+'''
+{prompt_to_test}
+'''
+"""
 
-@dataclass
-class Delimitation(ThresholdTest):
+
+@tags("llm", "zero_shot", "few_shot")
+@tasks("text_classification", "text_summarization")
+def Delimitation(model, min_threshold=7):
     """
     Evaluates the proper use of delimiters in prompts provided to Large Language Models.
 
@@ -63,78 +77,23 @@ class Delimitation(ThresholdTest):
     - The preset score threshold may not be refined enough for complex tasks and prompts, requiring regular manual
     adjustment.
     """
-
-    name = "delimitation"
-    required_inputs = ["model.prompt"]
-    default_params = {"min_threshold": 7}
-    tasks = ["text_classification", "text_summarization"]
-    tags = ["llm", "zero_shot", "few_shot"]
-
-    system_prompt = """
-You are a prompt evaluation AI. You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics. You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
-
-LLM Prompts that include different sections and user inputs should be properly delimitated. Ideally, the prompt should use triple quotes or backticks or at least single quotes around any user input, reference text or code block etc.
-This is to ensure that the prompt is parsed correctly by the model, different pieces of the prompt are understood as separate and any user-provided inputs are not interpreted as part of the prompt.
-Identify any issues in the user-submitted prompt and give a score from 1 to 10, where 10 is a perfect score, based on the number and severity of issues.
-
-Response Format:
-```
-Score: <score>
-Explanation: <explanation>
-```
-""".strip()
-    user_prompt = '''
-Prompt:
-"""
-{prompt_to_test}
-"""
-'''.strip()
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        result = results[0]
-        results_table = [
-            {
-                "Score": result.values["score"],
-                "Threshold": result.values["threshold"],
-                "Explanation": result.values["explanation"],
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pd.DataFrame(results_table),
-                    metadata=ResultTableMetadata(
-                        title="Delimination Test for LLM Prompt",
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        if not hasattr(self.inputs.model, "prompt"):
-            raise MissingRequiredTestInputError(missing_prompt_message)
-
-        response = call_model(
-            system_prompt=self.system_prompt,
-            user_prompt=self.user_prompt.format(
-                prompt_to_test=self.inputs.model.prompt.template
-            ),
-        )
-        score = get_score(response)
-        explanation = get_explanation(response)
-
-        passed = score > self.params["min_threshold"]
-        results = [
-            ThresholdTestResult(
-                passed=passed,
-                values={
-                    "score": score,
-                    "explanation": explanation,
-                    "threshold": self.params["min_threshold"],
-                },
-            )
-        ]
-
-        return self.cache_results(results, passed=passed)
+    if not hasattr(model, "prompt"):
+        raise MissingRequiredTestInputError(missing_prompt_message)
+
+    response = call_model(
+        system_prompt=SYSTEM,
+        user_prompt=USER.format(prompt_to_test=model.prompt.template),
+    )
+    score = get_score(response)
+    explanation = get_explanation(response)
+
+    passed = score > min_threshold
+
+    return [
+        {
+            "Score": score,
+            "Threshold": min_threshold,
+            "Explanation": explanation,
+            "Pass/Fail": "Pass" if passed else "Fail",
+        }
+    ], passed

validmind/tests/prompt_validation/NegativeInstruction.py

@@ -2,19 +2,8 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-from dataclasses import dataclass
-from typing import List
-
-import pandas as pd
-
+from validmind import tags, tasks
 from validmind.errors import MissingRequiredTestInputError
-from validmind.vm_models import (
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
 
 from .ai_powered_test import (
     call_model,
@@ -23,9 +12,47 @@ from .ai_powered_test import (
     missing_prompt_message,
 )
 
+SYSTEM = """
+You are a prompt evaluation AI.
+You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics.
+You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
+
+Consider the following documentation regarding negative instructions in prompts and utilize it to grade the user-submitted prompt:
+'''
+Best practices for LLM prompt engineering suggest that positive instructions should be preferred over negative instructions.
+For example, instead of saying "Don't do X", it is better to say "Do Y".
+This is because the model is more likely to generate the desired output if it is given a positive instruction.
+Prompts that are phrased in the affirmative, emphasizing what to do, tend to direct the LLM more clearly than those that focus on what not to do.
+Negative instructions can lead to ambiguities and undesired model responses.
+By emphasizing clarity and proactive guidance, we optimize the chances of obtaining relevant and targeted responses from the LLM.
+
+Example:
+Consider a scenario involving a chatbot designed to recommend movies.
+An instruction framed as, "Don't recommend movies that are horror or thriller", might cause the LLM to fixate on the genres mentioned,inadvertently producing undesired results.
+On the other hand, a positively-framed prompt like, "Recommend family-friendly movies or romantic comedies" provides clear guidance on the desired output.
+'''
+
+Based on this best practice, please score the user-submitted prompt on a scale of 1-10, where 10 is a perfect score.
+Provide an explanation for your score.
+
+Response Format:
+```
+Score: <score>
+Explanation: <explanation>
+```
+"""
+
+USER = """
+Prompt:
+'''
+{prompt_to_test}
+'''
+"""
+
 
-@dataclass
-class NegativeInstruction(ThresholdTest):
+@tags("llm", "zero_shot", "few_shot")
+@tasks("text_classification", "text_summarization")
+def NegativeInstruction(model, min_threshold=7):
     """
     Evaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts.
 
@@ -68,85 +95,24 @@ class NegativeInstruction(ThresholdTest):
     - The effectiveness of the test hinges significantly on the predetermined threshold level, which can be subjective
     and may need to be adjusted according to specific use-cases.
     """
-
-    name = "negative_instruction"
-    required_inputs = ["model.prompt"]
-    default_params = {"min_threshold": 7}
-    tasks = ["text_classification", "text_summarization"]
-    tags = ["llm", "zero_shot", "few_shot"]
-
-    system_prompt = """
-You are a prompt evaluation AI. You are aware of all prompt engineering best practices and can score prompts based on how well they satisfy different metrics. You analyse the prompts step-by-step based on provided documentation and provide a score and an explanation for how you produced that score.
-
-Consider the following documentation regarding negative instructions in prompts and utilize it to grade the user-submitted prompt:
-'''
-Best practices for LLM prompt engineering suggest that positive instructions should be preferred over negative instructions. For example, instead of saying "Don't do X", it is better to say "Do Y". This is because the model is more likely to generate the desired output if it is given a positive instruction.
-Prompts that are phrased in the affirmative, emphasizing what to do, tend to direct the LLM more clearly than those that focus on what not to do. Negative instructions can lead to ambiguities and undesired model responses. By emphasizing clarity and proactive guidance, we optimize the chances of obtaining relevant and targeted responses from the LLM.
-Example:
-Consider a scenario involving a chatbot designed to recommend movies. An instruction framed as, "Don't recommend movies that are horror or thriller" might cause the LLM to fixate on the genres mentioned, inadvertently producing undesired results. On the other hand, a positively-framed prompt like, "Recommend family-friendly movies or romantic comedies" provides clear guidance on the desired output.
-'''
-
-Based on this best practice, please score the user-submitted prompt on a scale of 1-10, where 10 is a perfect score.
-Provide an explanation for your score.
-
-Response Format:
-```
-Score: <score>
-Explanation: <explanation>
-```
-""".strip()
-    user_prompt = '''
-Prompt:
-"""
-{prompt_to_test}
-"""
-'''.strip()
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        result = results[0]
-        results_table = [
-            {
-                "Score": result.values["score"],
-                "Threshold": result.values["threshold"],
-                "Explanation": result.values["explanation"],
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-        ]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=pd.DataFrame(results_table),
-                    metadata=ResultTableMetadata(
-                        title="Negative Instruction Test on Prompt",
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        if not hasattr(self.inputs.model, "prompt"):
-            raise MissingRequiredTestInputError(missing_prompt_message)
-
-        response = call_model(
-            system_prompt=self.system_prompt,
-            user_prompt=self.user_prompt.format(
-                prompt_to_test=self.inputs.model.prompt.template
-            ),
-        )
-        score = get_score(response)
-        explanation = get_explanation(response)
-
-        passed = score > self.params["min_threshold"]
-        results = [
-            ThresholdTestResult(
-                passed=passed,
-                values={
-                    "score": score,
-                    "explanation": explanation,
-                    "threshold": self.params["min_threshold"],
-                },
-            )
-        ]
-
-        return self.cache_results(results, passed=passed)
+    if not hasattr(model, "prompt"):
+        raise MissingRequiredTestInputError(missing_prompt_message)
+
+    response = call_model(
+        system_prompt=SYSTEM,
+        user_prompt=USER.format(prompt_to_test=model.prompt.template),
+    )
+    score = get_score(response)
+    explanation = get_explanation(response)
+
+    passed = score > min_threshold
+    result = [
+        {
+            "Score": score,
+            "Threshold": min_threshold,
+            "Explanation": explanation,
+            "Pass/Fail": "Pass" if passed else "Fail",
+        }
+    ]
+
+    return result, passed