validmind 2.5.8__py3-none-any.whl → 2.5.15__py3-none-any.whl

validmind/tests/prompt_validation/Conciseness.py

@@ -29,31 +29,33 @@ class Conciseness(ThresholdTest):
     """
     Analyzes and grades the conciseness of prompts provided to a Large Language Model.
 
-    **Purpose:**
+    ### Purpose
+
     The Conciseness Assessment is designed to evaluate the brevity and succinctness of prompts provided to a Language
     Learning Model (LLM). A concise prompt strikes a balance between offering clear instructions and eliminating
     redundant or unnecessary information, ensuring that the LLM receives relevant input without being overwhelmed.
 
-    **Test Mechanism:**
+    ### Test Mechanism
+
     Using an LLM, this test conducts a conciseness analysis on input prompts. The analysis grades the prompt on a scale
     from 1 to 10, where the grade reflects how well the prompt delivers clear instructions without being verbose.
     Prompts that score equal to or above a predefined threshold (default set to 7) are deemed successfully concise.
     This threshold can be adjusted to meet specific requirements.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
     - Prompts that consistently score below the predefined threshold.
     - Prompts that are overly wordy or contain unnecessary information.
     - Prompts that create confusion or ambiguity due to excess or unnecessary information.
 
-    **Strengths:**
+    ### Strengths
 
     - Ensures clarity and effectiveness of the prompts.
     - Promotes brevity and preciseness in prompts without sacrificing essential information.
     - Useful for models like LLMs, where input prompt length and clarity greatly influence model performance.
     - Provides a quantifiable measure of prompt conciseness.
 
-    **Limitations:**
+    ### Limitations
 
     - The conciseness score is based on an AI's assessment, which might not fully capture human interpretation of
     conciseness.

validmind/tests/prompt_validation/Delimitation.py

@@ -29,38 +29,39 @@ class Delimitation(ThresholdTest):
     """
     Evaluates the proper use of delimiters in prompts provided to Large Language Models.
 
-    **Purpose:**
-    This test, dubbed the "Delimitation Test", is engineered to assess whether prompts provided to the Language
-    Learning Model (LLM) correctly use delimiters to mark different sections of the input. Well-delimited prompts
-    simplify the interpretation process for LLM, ensuring responses are precise and accurate.
+    ### Purpose
+
+    The Delimitation Test aims to assess whether prompts provided to the Language Learning Model (LLM) correctly use
+    delimiters to mark different sections of the input. Well-delimited prompts help simplify the interpretation process
+    for the LLM, ensuring that the responses are precise and accurate.
+
+    ### Test Mechanism
 
-    **Test Mechanism:**
     The test employs an LLM to examine prompts for appropriate use of delimiters such as triple quotation marks, XML
-    tags, and section titles. Each prompt is assigned a score from 1 to 10 based on its delimitation integrity. Those
+    tags, and section titles. Each prompt is assigned a score from 1 to 10 based on its delimitation integrity. Prompts
     with scores equal to or above the preset threshold (which is 7 by default, although it can be adjusted as
     necessary) pass the test.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
-    - The test identifies prompts where a delimiter is missing, improperly placed, or incorrect, which can lead to
-    misinterpretation by the LLM.
-    - A high-risk scenario may involve complex prompts with multiple tasks or diverse data where correct delimitation
-    is integral to understanding.
-    - Low scores (below the threshold) are a clear indicator of high risk.
+    - Prompts missing, improperly placed, or incorrectly used delimiters, leading to misinterpretation by the LLM.
+    - High-risk scenarios with complex prompts involving multiple tasks or diverse data where correct delimitation is
+    crucial.
+    - Scores below the threshold, indicating a high risk.
 
-    **Strengths:**
+    ### Strengths
 
-    - This test ensures clarity in the demarcation of different components of given prompts.
-    - It helps reduce ambiguity in understanding prompts, particularly for complex tasks.
-    - Scoring allows for quantified insight into the appropriateness of delimiter usage, aiding continuous improvement.
+    - Ensures clarity in demarcating different components of given prompts.
+    - Reduces ambiguity in understanding prompts, especially for complex tasks.
+    - Provides a quantified insight into the appropriateness of delimiter usage, aiding continuous improvement.
 
-    **Limitations:**
+    ### Limitations
 
-    - The test only checks for the presence and placement of delimiter, not whether the correct delimiter type is used
-    for the specific data or task.
-    - It may not fully reveal the impacts of poor delimitation on LLM's final performance.
-    - Depending on the complexity of the tasks and prompts, the preset score threshold may not be refined enough,
-    requiring regular manual adjustment.
+    - Only checks for the presence and placement of delimiters, not whether the correct delimiter type is used for the
+    specific data or task.
+    - May not fully reveal the impacts of poor delimitation on the LLM's final performance.
+    - The preset score threshold may not be refined enough for complex tasks and prompts, requiring regular manual
+    adjustment.
     """
 
     name = "delimitation"

validmind/tests/prompt_validation/NegativeInstruction.py

@@ -29,34 +29,36 @@ class NegativeInstruction(ThresholdTest):
     """
     Evaluates and grades the use of affirmative, proactive language over negative instructions in LLM prompts.
 
-    **Purpose:**
+    ### Purpose
+
     The Negative Instruction test is utilized to scrutinize the prompts given to a Large Language Model (LLM). The
     objective is to ensure these prompts are expressed using proactive, affirmative language. The focus is on
     instructions indicating what needs to be done rather than what needs to be avoided, thereby guiding the LLM more
     efficiently towards the desired output.
 
-    **Test Mechanism:**
+    ### Test Mechanism
+
     An LLM is employed to evaluate each prompt. The prompt is graded based on its use of positive instructions with
     scores ranging between 1-10. This grade reflects how effectively the prompt leverages affirmative language while
     shying away from negative or restrictive instructions. A prompt that attains a grade equal to or above a
     predetermined threshold (7 by default) is regarded as adhering effectively to the best practices of positive
     instruction. This threshold can be custom-tailored through the test parameters.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
     - Low score obtained from the LLM analysis, indicating heavy reliance on negative instructions in the prompts.
     - Failure to surpass the preset minimum threshold.
     - The LLM generates ambiguous or undesirable outputs as a consequence of the negative instructions used in the
     prompt.
 
-    **Strengths:**
+    ### Strengths
 
     - Encourages the usage of affirmative, proactive language in prompts, aiding in more accurate and advantageous
     model responses.
     - The test result provides a comprehensible score, helping to understand how well a prompt follows the positive
     instruction best practices.
 
-    **Limitations:**
+    ### Limitations
 
     - Despite an adequate score, a prompt could still be misleading or could lead to undesired responses due to factors
     not covered by this test.

validmind/tests/prompt_validation/Robustness.py

@@ -24,31 +24,33 @@ class Robustness(ThresholdTest):
     """
     Assesses the robustness of prompts provided to a Large Language Model under varying conditions and contexts.
 
-    **Purpose:**
+    ### Purpose
+
     The Robustness test is meant to evaluate the resilience and reliability of prompts provided to a Language Learning
-    Model (LLM). The aim of this test is to guarantee that the prompts consistently generate accurate and the expected
-    outputs, despite being in diverse or challenging scenarios.
+    Model (LLM). The aim of this test is to guarantee that the prompts consistently generate accurate and expected
+    outputs, even in diverse or challenging scenarios.
+
+    ### Test Mechanism
 
-    **Test Mechanism:**
     The Robustness test appraises prompts under various conditions, alterations, and contexts to ascertain their
-    stability in producing consistent responses from the LLM. Factors evaluated range from different phrasings,
-    inclusion of potential distracting elements, and various input complexities. By default, the test generates 10
-    inputs for a prompt but can be adjusted according to test parameters.
+    stability in producing consistent responses from the LLM. Factors evaluated include different phrasings, inclusion
+    of potential distracting elements, and various input complexities. By default, the test generates 10 inputs for a
+    prompt but can be adjusted according to test parameters.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
     - If the output from the tests diverges extensively from the expected results, this indicates high risk.
     - When the prompt doesn't give a consistent performance across various tests.
     - A high risk is indicated when the prompt is susceptible to breaking, especially when the output is expected to be
     of a specific type.
 
-    **Strengths:**
+    ### Strengths
 
     - The robustness test helps to ensure stable performance of the LLM prompts and lowers the chances of generating
     unexpected or off-target outputs.
     - This test is vital for applications where predictability and reliability of the LLM’s output are crucial.
 
-    **Limitations:**
+    ### Limitations
 
     - Currently, the test only supports single-variable prompts, which restricts its application to more complex models.
     - When there are too many target classes (over 10), the test is skipped, which can leave potential vulnerabilities

validmind/tests/prompt_validation/Specificity.py

@@ -27,40 +27,42 @@ from .ai_powered_test import (
 @dataclass
 class Specificity(ThresholdTest):
     """
-    Evaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity,
-    detail, and relevance.
+    Evaluates and scores the specificity of prompts provided to a Large Language Model (LLM), based on clarity, detail,
+    and relevance.
+
+    ### Purpose
 
-    **Purpose:**
     The Specificity Test evaluates the clarity, precision, and effectiveness of the prompts provided to a Language
-    Learning Model (LLM). It aims to ensure that the instructions embedded in a prompt are indisputably clear and
-    relevant, thereby helping to yank out ambiguity and steer the LLM towards desired outputs. This level of
-    specificity significantly affects the accuracy and relevance of LLM outputs.
+    Model (LLM). It aims to ensure that the instructions embedded in a prompt are indisputably clear and relevant,
+    thereby helping to remove ambiguity and steer the LLM towards desired outputs. This level of specificity
+    significantly affects the accuracy and relevance of LLM outputs.
+
+    ### Test Mechanism
 
-    **Test Mechanism:**
     The Specificity Test employs an LLM to grade each prompt based on clarity, detail, and relevance parameters within
     a specificity scale that extends from 1 to 10. On this scale, prompts scoring equal to or more than a predefined
     threshold (set to 7 by default) pass the evaluation, while those scoring below this threshold fail it. Users can
     adjust this threshold as per their requirements.
 
-    **Signs of High Risk:**
+    ### Signs of High Risk
 
     - Prompts scoring consistently below the established threshold
     - Vague or ambiguous prompts that do not provide clear direction to the LLM
     - Overly verbose prompts that may confuse the LLM instead of providing clear guidance
 
-    **Strengths:**
+    ### Strengths
 
     - Enables precise and clear communication with the LLM to achieve desired outputs
     - Serves as a crucial means to measure the effectiveness of prompts
     - Highly customizable, allowing users to set their threshold based on specific use cases
 
-    **Limitations:**
+    ### Limitations
 
     - This test doesn't consider the content comprehension capability of the LLM
     - High specificity score doesn't guarantee a high-quality response from the LLM, as the model's performance is also
     dependent on various other factors
     - Striking a balance between specificity and verbosity can be challenging, as overly detailed prompts might confuse
-    or mislead the model.
+    or mislead the model
     """
 
     name = "specificity"

validmind/tests/prompt_validation/ai_powered_test.py

@@ -5,6 +5,7 @@
 import re
 
 from validmind.ai.utils import get_client_and_model
+from validmind.client_config import client_config
 
 missing_prompt_message = """
 Cannot run prompt validation tests on a model with no prompt.
@@ -24,6 +25,11 @@ def call_model(
     system_prompt: str, user_prompt: str, temperature: float = 0.0, seed: int = 42
 ):
     """Call LLM with the given prompts and return the response"""
+    if not client_config.can_generate_llm_test_descriptions():
+        raise ValueError(
+            "LLM based descriptions are not enabled for your organization."
+        )
+
     client, model = get_client_and_model()
 
     return (

validmind/tests/run.py

@@ -17,6 +17,7 @@ from validmind.vm_models import (
     MetricResult,
     ResultSummary,
     ResultTable,
+    ResultTableMetadata,
     TestContext,
     TestInput,
     ThresholdTestResults,
@@ -147,6 +148,26 @@ def _combine_figures(figure_lists: List[List[Any]], input_groups: List[Dict[str,
     return [figure for figures in figure_lists for figure in figures]
 
 
+def _combine_unit_metrics(results: List[MetricResultWrapper]):
+    if not results[0].scalar:
+        return
+
+    for result in results:
+        table = ResultTable(
+            data=[{"value": result.scalar}],
+            metadata=ResultTableMetadata(title="Unit Metrics"),
+        )
+        if not result.metric:
+            result.metric = MetricResult(
+                ref_id="will_be_overwritten",
+                key=result.result_id,
+                value=result.scalar,
+                summary=ResultSummary(results=[table]),
+            )
+        else:
+            result.metric.summary.results.append(table)
+
+
 def metric_comparison(
     results: List[MetricResultWrapper],
     test_id: TestID,
@@ -172,22 +193,41 @@ def metric_comparison(
                 raise ValueError(f"Unsupported type for value: {v}")
         input_group_strings.append(new_group)
 
-    merged_summary = _combine_summaries(
-        [
-            {"inputs": input_group_strings[i], "summary": result.metric.summary}
-            for i, result in enumerate(results)
-        ]
-    )
-    merged_figures = _combine_figures(
-        [result.figures for result in results], input_groups
-    )
-
-    # Patch figure metadata so they are connected to the comparison result
-    if merged_figures and len(merged_figures):
-        for i, figure in enumerate(merged_figures):
-            figure.key = f"{figure.key}-{i}"
-            figure.metadata["_name"] = test_id
-            figure.metadata["_ref_id"] = ref_id
+    # handle unit metrics (scalar values) by adding it to the summary
+    _combine_unit_metrics(results)
+
+    # Check if the results list contains a result object with a metric
+    if any(
+        hasattr(result, "metric")
+        and hasattr(result.metric, "summary")
+        and result.metric.summary
+        for result in results
+    ):
+        # Compute merged summaries only if there is a result with a metric
+        merged_summary = _combine_summaries(
+            [
+                {"inputs": input_group_strings[i], "summary": result.metric.summary}
+                for i, result in enumerate(results)
+            ]
+        )
+    else:
+        merged_summary = None
+
+    # Check if the results list contains a result object with figures
+    if any(hasattr(result, "figures") and result.figures for result in results):
+        # Compute merged figures only if there is at least one result with figures
+        merged_figures = _combine_figures(
+            [result.figures for result in results],
+            input_groups,
+        )
+        # Patch figure metadata so they are connected to the comparison result
+        if merged_figures and len(merged_figures):
+            for i, figure in enumerate(merged_figures):
+                figure.key = f"{figure.key}-{i}"
+                figure.metadata["_name"] = test_id
+                figure.metadata["_ref_id"] = ref_id
+    else:
+        merged_figures = None
 
     return MetricResultWrapper(
         result_id=test_id,
@@ -196,7 +236,7 @@ def metric_comparison(
                 test_id=test_id,
                 default_description=f"Comparison test result for {test_id}",
                 summary=merged_summary.serialize() if merged_summary else None,
-                figures=merged_figures,
+                figures=merged_figures if merged_figures else None,
                 should_generate=generate_description,
             ),
         ],
@@ -294,6 +334,8 @@ def threshold_test_comparison(
 def run_comparison_test(
     test_id: TestID,
     input_grid: Union[Dict[str, List[Any]], List[Dict[str, Any]]],
+    name: str = None,
+    unit_metrics: List[TestID] = None,
     params: Dict[str, Any] = None,
     show: bool = True,
     output_template: str = None,
@@ -308,6 +350,8 @@ def run_comparison_test(
     results = [
         run_test(
             test_id,
+            name=name,
+            unit_metrics=unit_metrics,
             inputs=inputs,
             show=False,
             params=params,
@@ -387,33 +431,34 @@ def run_test(
             "When providing an `input_grid`, you cannot also provide `inputs` or `kwargs`"
         )
 
+    if unit_metrics:
+        metric_id_name = "".join(word[0].upper() + word[1:] for word in name.split())
+        test_id = f"validmind.composite_metric.{metric_id_name}" or test_id
+
     if input_grid:
         return run_comparison_test(
             test_id,
             input_grid,
+            name=name,
+            unit_metrics=unit_metrics,
             params=params,
             output_template=output_template,
             show=show,
             generate_description=__generate_description,
         )
 
-    if test_id and test_id.startswith("validmind.unit_metrics"):
+    if test_id.startswith("validmind.unit_metrics"):
         # TODO: as we move towards a more unified approach to metrics
         # we will want to make everything functional and remove the
         # separation between unit metrics and "normal" metrics
         return run_metric(test_id, inputs=inputs, params=params, show=show)
 
     if unit_metrics:
-        metric_id_name = "".join(word[0].upper() + word[1:] for word in name.split())
-        test_id = f"validmind.composite_metric.{metric_id_name}"
-
         error, TestClass = load_composite_metric(
             unit_metrics=unit_metrics, metric_name=metric_id_name
         )
-
         if error:
             raise LoadTestError(error)
-
     else:
         TestClass = load_test(test_id, reload=True)