validmind 2.8.20__py3-none-any.whl → 2.8.26__py3-none-any.whl

validmind/__init__.py

@@ -53,6 +53,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
+from .experimental import agents as experimental_agent
 from .tests.decorator import tags, tasks, test
 from .tests.run import print_env
 from .utils import is_notebook, parse_version
@@ -126,4 +127,6 @@ __all__ = [  # noqa
     "unit_metrics",
     "test_suites",
     "log_text",
+    # experimental features
+    "experimental_agent",
 ]

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.8.20"
+__version__ = "2.8.25"

validmind/ai/utils.py

@@ -15,6 +15,10 @@ logger = get_logger(__name__)
 
 __client = None
 __model = None
+__judge_llm = None
+__judge_embeddings = None
+EMBEDDINGS_MODEL = "text-embedding-3-small"
+
 # can be None, True or False (ternary to represent initial state, ack and failed ack)
 __ack = None
 
@@ -105,6 +109,91 @@ def get_client_and_model():
     return __client, __model
 
 
+def get_judge_config(judge_llm=None, judge_embeddings=None):
+    try:
+        from langchain_core.embeddings import Embeddings
+        from langchain_core.language_models.chat_models import BaseChatModel
+        from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+
+        from validmind.models.function import FunctionModel
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
+    if judge_llm is not None or judge_embeddings is not None:
+        if isinstance(judge_llm, FunctionModel) and judge_llm is not None:
+            if isinstance(judge_llm.model, BaseChatModel):
+                judge_llm = judge_llm.model
+            else:
+                raise ValueError(
+                    "The ValidMind Functional model provided does not have have a langchain compatible LLM as a model attribute."
+                    "To use default ValidMind LLM, do not set judge_llm/judge_embedding parameter, "
+                    "ensure that you are connected to the ValidMind API and confirm ValidMind AI is enabled for your account."
+                )
+        if isinstance(judge_embeddings, FunctionModel) and judge_embeddings is not None:
+            if isinstance(judge_llm.model, BaseChatModel):
+                judge_embeddings = judge_embeddings.model
+            else:
+                raise ValueError(
+                    "The ValidMind Functional model provided does not have have a langchain compatible embeddings model as a model attribute."
+                    "To use default ValidMind LLM, do not set judge_embedding parameter, "
+                    "ensure that you are connected to the ValidMind API and confirm ValidMind AI is enabled for your account."
+                )
+
+        if (isinstance(judge_llm, BaseChatModel) or judge_llm is None) and (
+            isinstance(judge_embeddings, Embeddings) or judge_embeddings is None
+        ):
+            return judge_llm, judge_embeddings
+        else:
+            raise ValueError(
+                "Provided Judge LLM/Embeddings are not Langchain compatible. Ensure the judge LLM/embedding provided are an instance of "
+                "Langchain BaseChatModel and LangchainEmbeddings.  To use default ValidMind LLM, do not set judge_llm/judge_embedding parameter, "
+                "ensure that you are connected to the ValidMind API and confirm ValidMind AI is enabled for your account."
+            )
+
+    # grab default values if not passed at run time
+    global __judge_llm, __judge_embeddings
+    if __judge_llm and __judge_embeddings:
+        return __judge_llm, __judge_embeddings
+
+    client, model = get_client_and_model()
+    os.environ["OPENAI_API_BASE"] = str(client.base_url)
+
+    __judge_llm = ChatOpenAI(api_key=client.api_key, model=model)
+    __judge_embeddings = OpenAIEmbeddings(
+        api_key=client.api_key, model=EMBEDDINGS_MODEL
+    )
+
+    return __judge_llm, __judge_embeddings
+
+
+def set_judge_config(judge_llm, judge_embeddings):
+    global __judge_llm, __judge_embeddings
+    try:
+        from langchain_core.embeddings import Embeddings
+        from langchain_core.language_models.chat_models import BaseChatModel
+
+        from validmind.models.function import FunctionModel
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+    if isinstance(judge_llm, BaseChatModel) and isinstance(
+        judge_embeddings, Embeddings
+    ):
+        __judge_llm = judge_llm
+        __judge_embeddings = judge_embeddings
+        # Assuming 'your_object' is the object you want to check
+    elif isinstance(judge_llm, FunctionModel) and isinstance(
+        judge_embeddings, FunctionModel
+    ):
+        __judge_llm = judge_llm.model
+        __judge_embeddings = judge_embeddings.model
+    else:
+        raise ValueError(
+            "Provided Judge LLM/Embeddings are not Langchain compatible. Ensure the judge LLM/embedding provided are an instance of "
+            "Langchain BaseChatModel and LangchainEmbeddings. To use default ValidMind LLM, do not set judge_llm/judge_embedding parameter, "
+            "ensure that you are connected to the ValidMind API and confirm ValidMind AI is enabled for your account."
+        )
+
+
 def is_configured():
     global __ack
 

validmind/api_client.py

@@ -448,6 +448,7 @@ async def alog_metric(
     params: Optional[Dict[str, Any]] = None,
     recorded_at: Optional[str] = None,
     thresholds: Optional[Dict[str, Any]] = None,
+    passed: Optional[bool] = None,
 ):
     """See log_metric for details."""
     if not key or not isinstance(key, str):
@@ -476,6 +477,7 @@ async def alog_metric(
                     "params": params or {},
                     "recorded_at": recorded_at,
                     "thresholds": thresholds or {},
+                    "passed": passed if passed is not None else None,
                 },
                 cls=NumpyEncoder,
                 allow_nan=False,
@@ -493,6 +495,7 @@ def log_metric(
     params: Optional[Dict[str, Any]] = None,
     recorded_at: Optional[str] = None,
     thresholds: Optional[Dict[str, Any]] = None,
+    passed: Optional[bool] = None,
 ):
     """Logs a unit metric.
 
@@ -518,6 +521,7 @@ def log_metric(
         params=params,
         recorded_at=recorded_at,
         thresholds=thresholds,
+        passed=passed,
     )
 
 

validmind/client.py

@@ -61,6 +61,7 @@ def init_dataset(
     class_labels: Optional[Dict[str, Any]] = None,
     type: Optional[str] = None,
     input_id: Optional[str] = None,
+    copy_data: bool = True,
     __log: bool = True,
 ) -> VMDataset:
     """
@@ -92,6 +93,7 @@ def init_dataset(
             this will be set to `dataset` but if you are passing this dataset as a
             test input using some other key than `dataset`, then you should set
             this to the same key.
+        copy_data (bool, optional): Whether to copy the data. Defaults to True.
         __log (bool): Whether to log the input. Defaults to True.
 
     Raises:
@@ -121,6 +123,7 @@ def init_dataset(
             extra_columns=extra_columns,
             target_class_labels=class_labels,
             date_time_index=date_time_index,
+            copy_data=copy_data,
         )
     elif isinstance(dataset, pl.DataFrame):
         vm_dataset = PolarsDataset(

validmind/experimental/agents.py

@@ -0,0 +1,65 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Agent interface for all text generation tasks
+"""
+
+import requests
+
+from validmind.api_client import _get_api_headers, _get_url, raise_api_error
+from validmind.utils import is_html, md_to_html
+from validmind.vm_models.result import TextGenerationResult
+
+
+def run_task(
+    task: str,
+    input: dict,
+    show: bool = True,
+) -> TextGenerationResult:
+    """
+    Run text generation tasks using AI models.
+
+    Args:
+        task (str): Type of text generation task to run. Currently supports:
+            - 'code_explainer': Generates natural language explanations of code
+        input (dict): Input parameters for the generation task:
+            - For code_explainer: Must contain 'source_code' and optional parameters
+        show (bool): Whether to display the generated result. Defaults to True.
+
+    Returns:
+        TextGenerationResult: Result object containing the generated text and metadata
+
+    Raises:
+        ValueError: If an unsupported task is provided
+        requests.exceptions.RequestException: If the API request fails
+    """
+    if task == "code_explainer" or task == "qualitative_text_generation":
+        r = requests.post(
+            url=_get_url(f"ai/generate/{task}"),
+            headers=_get_api_headers(),
+            json=input,
+        )
+
+        if r.status_code != 200:
+            raise_api_error(r.text)
+
+        generated_text = r.json()["content"]
+    else:
+        raise ValueError(f"Unsupported task: {task}")
+
+    if not is_html(generated_text):
+        generated_text = md_to_html(generated_text, mathml=True)
+
+    # Create a test result with the generated text
+    result = TextGenerationResult(
+        result_type=f"{task}",
+        description=generated_text,
+        title=f"Text Generation: {task}",
+        doc=f"Generated {task}",
+    )
+    if show:
+        result.show()
+
+    return result

validmind/template.py

@@ -53,8 +53,9 @@ def _convert_sections_to_section_tree(
 
     if start_section_id and not section_tree:
         raise ValueError(f"Section {start_section_id} not found in template")
-
-    return sorted(section_tree, key=lambda x: x.get("order", 0))
+    # sort the section tree by the order of the sections in the template (if provided)
+    # set the order to 9999 for the sections that do not have an order
+    return sorted(section_tree, key=lambda x: x.get("order", 9999))
 
 
 def _create_content_widget(content: Dict[str, Any]) -> Widget:

validmind/tests/data_validation/MutualInformation.py

@@ -68,8 +68,20 @@ def MutualInformation(
     if task not in ["classification", "regression"]:
         raise ValueError("task must be either 'classification' or 'regression'")
 
-    X = dataset.x
-    y = dataset.y
+    # Check if numeric features exist
+    if not dataset.feature_columns_numeric:
+        raise ValueError(
+            "No numeric features found in dataset. Mutual Information test requires numeric features."
+        )
+
+    # Check if target column exists
+    if not dataset.target_column:
+        raise ValueError(
+            "Target column is required for Mutual Information calculation but was not provided."
+        )
+
+    X = dataset._df[dataset.feature_columns_numeric]
+    y = dataset._df[dataset.target_column]
 
     # Select appropriate MI function based on task type
     if task == "classification":

validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -34,6 +34,8 @@ def AnswerCorrectness(
     user_input_column="user_input",
     response_column="response",
     reference_column="reference",
+    judge_llm=None,
+    judge_embeddings=None,
 ):
     """
     Evaluates the correctness of answers in a dataset with respect to the provided ground
@@ -118,7 +120,9 @@ def AnswerCorrectness(
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[answer_correctness()], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=[answer_correctness()],
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     score_column = "answer_correctness"

validmind/tests/model_validation/ragas/AspectCritic.py

@@ -51,6 +51,8 @@ def AspectCritic(
         "maliciousness",
     ],
     additional_aspects: list = None,
+    judge_llm=None,
+    judge_embeddings=None,
 ):
     """
     Evaluates generations against the following aspects: harmfulness, maliciousness,
@@ -158,7 +160,9 @@ def AspectCritic(
     all_aspects = [built_in_aspects[aspect] for aspect in aspects] + custom_aspects
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=all_aspects, **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=all_aspects,
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     # reverse the score for aspects where lower is better

validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -33,6 +33,8 @@ def ContextEntityRecall(
     dataset,
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
+    judge_llm=None,
+    judge_embeddings=None,
 ):
     """
     Evaluates the context entity recall for dataset entries and visualizes the results.
@@ -113,7 +115,9 @@ def ContextEntityRecall(
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_entity_recall()], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=[context_entity_recall()],
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     score_column = "context_entity_recall"

validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -34,6 +34,8 @@ def ContextPrecision(
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
+    judge_llm=None,
+    judge_embeddings=None,
 ):  # noqa: B950
     """
     Context Precision is a metric that evaluates whether all of the ground-truth
@@ -109,7 +111,9 @@ def ContextPrecision(
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_precision()], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=[context_precision()],
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     score_column = "llm_context_precision_with_reference"

validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py

@@ -34,6 +34,8 @@ def ContextPrecisionWithoutReference(
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     response_column: str = "response",
+    judge_llm=None,
+    judge_embeddings=None,
 ):  # noqa: B950
     """
     Context Precision Without Reference is a metric used to evaluate the relevance of
@@ -104,7 +106,9 @@ def ContextPrecisionWithoutReference(
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_precision()], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=[context_precision()],
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     score_column = "llm_context_precision_without_reference"

validmind/tests/model_validation/ragas/ContextRecall.py

@@ -34,6 +34,8 @@ def ContextRecall(
     user_input_column: str = "user_input",
     retrieved_contexts_column: str = "retrieved_contexts",
     reference_column: str = "reference",
+    judge_llm=None,
+    judge_embeddings=None,
 ):
     """
     Context recall measures the extent to which the retrieved context aligns with the
@@ -109,7 +111,9 @@ def ContextRecall(
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_recall()], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=[context_recall()],
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     score_column = "context_recall"

validmind/tests/model_validation/ragas/Faithfulness.py

@@ -34,6 +34,8 @@ def Faithfulness(
     user_input_column="user_input",
     response_column="response",
     retrieved_contexts_column="retrieved_contexts",
+    judge_llm=None,
+    judge_embeddings=None,
 ):  # noqa
     """
     Evaluates the faithfulness of the generated answers with respect to retrieved contexts.
@@ -114,7 +116,9 @@ def Faithfulness(
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[faithfulness()], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=[faithfulness()],
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     score_column = "faithfulness"

validmind/tests/model_validation/ragas/NoiseSensitivity.py

@@ -38,6 +38,8 @@ def NoiseSensitivity(
     reference_column="reference",
     focus="relevant",
     user_input_column="user_input",
+    judge_llm=None,
+    judge_embeddings=None,
 ):
     """
     Assesses the sensitivity of a Large Language Model (LLM) to noise in retrieved context by measuring how often it
@@ -149,7 +151,7 @@ def NoiseSensitivity(
     result_df = evaluate(
         Dataset.from_pandas(df),
         metrics=[noise_sensitivity(focus=focus)],
-        **get_ragas_config(),
+        **get_ragas_config(judge_llm, judge_embeddings),
     ).to_pandas()
 
     score_column = f"noise_sensitivity_{focus}"

validmind/tests/model_validation/ragas/ResponseRelevancy.py

@@ -34,6 +34,8 @@ def ResponseRelevancy(
     user_input_column="user_input",
     retrieved_contexts_column=None,
     response_column="response",
+    judge_llm=None,
+    judge_embeddings=None,
 ):
     """
     Assesses how pertinent the generated answer is to the given prompt.
@@ -44,8 +46,8 @@ def ResponseRelevancy(
     relevancy. This metric is computed using the `user_input`, the `retrieved_contexts`
     and the `response`.
 
-    The Response Relevancy is defined as the mean cosine similartiy of the original
-    `user_input` to a number of artifical questions, which are generated (reverse-engineered)
+    The Response Relevancy is defined as the mean cosine similarity of the original
+    `user_input` to a number of artificial questions, which are generated (reverse-engineered)
     based on the `response`:
 
     $$
@@ -62,7 +64,7 @@ def ResponseRelevancy(
 
     **Note**: *This is a reference-free metric, meaning that it does not require a
     `ground_truth` answer to compare against. A similar metric that does evaluate the
-    correctness of a generated answser with respect to a `ground_truth` answer is
+    correctness of a generated answers with respect to a `ground_truth` answer is
     `validmind.model_validation.ragas.AnswerCorrectness`.*
 
     ### Configuring Columns
@@ -128,7 +130,7 @@ def ResponseRelevancy(
     result_df = evaluate(
         Dataset.from_pandas(df),
         metrics=metrics,
-        **get_ragas_config(),
+        **get_ragas_config(judge_llm, judge_embeddings),
     ).to_pandas()
 
     score_column = "answer_relevancy"

validmind/tests/model_validation/ragas/SemanticSimilarity.py

@@ -33,6 +33,8 @@ def SemanticSimilarity(
     dataset,
     response_column="response",
     reference_column="reference",
+    judge_llm=None,
+    judge_embeddings=None,
 ):
     """
     Calculates the semantic similarity between generated responses and ground truths
@@ -107,7 +109,9 @@ def SemanticSimilarity(
     df = get_renamed_columns(dataset._df, required_columns)
 
     result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[semantic_similarity()], **get_ragas_config()
+        Dataset.from_pandas(df),
+        metrics=[semantic_similarity()],
+        **get_ragas_config(judge_llm, judge_embeddings)
     ).to_pandas()
 
     score_column = "semantic_similarity"

validmind/tests/model_validation/ragas/utils.py

@@ -2,34 +2,14 @@
 # See the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
-import os
-
-from validmind.ai.utils import get_client_and_model, is_configured
+from validmind.ai.utils import get_judge_config
 
 EMBEDDINGS_MODEL = "text-embedding-3-small"
 
 
-def get_ragas_config():
-    # import here since its an optional dependency
-    try:
-        from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-    except ImportError:
-        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
-
-    if not is_configured():
-        raise ValueError(
-            "LLM is not configured. Please set an `OPENAI_API_KEY` environment variable "
-            "or ensure that you are connected to the ValidMind API and ValidMind AI is "
-            "enabled for your account."
-        )
-
-    client, model = get_client_and_model()
-    os.environ["OPENAI_API_BASE"] = str(client.base_url)
-
-    return {
-        "llm": ChatOpenAI(api_key=client.api_key, model=model),
-        "embeddings": OpenAIEmbeddings(api_key=client.api_key, model=EMBEDDINGS_MODEL),
-    }
+def get_ragas_config(judge_llm=None, judge_embeddings=None):
+    judge_llm, judge_embeddings = get_judge_config(judge_llm, judge_embeddings)
+    return {"llm": judge_llm, "embeddings": judge_embeddings}
 
 
 def make_sub_col_udf(root_col, sub_col):

validmind/tests/model_validation/sklearn/OverfitDiagnosis.py

@@ -220,6 +220,16 @@ def OverfitDiagnosis(
     - May not capture more subtle forms of overfitting that do not exceed the threshold.
     - Assumes that the binning of features adequately represents the data segments.
     """
+
+    numeric_and_categorical_feature_columns = (
+        datasets[0].feature_columns_numeric + datasets[0].feature_columns_categorical
+    )
+
+    if not numeric_and_categorical_feature_columns:
+        raise ValueError(
+            "No valid numeric or categorical columns found in features_columns"
+        )
+
     is_classification = bool(datasets[0].probability_column(model))
 
     if not metric:
@@ -246,7 +256,7 @@ def OverfitDiagnosis(
     figures = []
     results_headers = ["slice", "shape", "feature", metric]
 
-    for feature_column in datasets[0].feature_columns:
+    for feature_column in numeric_and_categorical_feature_columns:
         bins = 10
         if feature_column in datasets[0].feature_columns_categorical:
             bins = len(train_df[feature_column].unique())

validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py

@@ -211,6 +211,19 @@ def WeakspotsDiagnosis(
     improvement.
     """
     feature_columns = features_columns or datasets[0].feature_columns
+    numeric_and_categorical_columns = (
+        datasets[0].feature_columns_numeric + datasets[0].feature_columns_categorical
+    )
+
+    feature_columns = [
+        col for col in feature_columns if col in numeric_and_categorical_columns
+    ]
+
+    if not feature_columns:
+        raise ValueError(
+            "No valid numeric or categorical columns found in features_columns"
+        )
+
     if not all(col in datasets[0].feature_columns for col in feature_columns):
         raise ValueError(
             "Column(s) provided in features_columns do not exist in the dataset"

validmind/tests/prompt_validation/Bias.py

@@ -45,7 +45,7 @@ Prompt:
 
 @tags("llm", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Bias(model, min_threshold=7):
+def Bias(model, min_threshold=7, judge_llm=None):
     """
     Assesses potential bias in a Large Language Model by analyzing the distribution and order of exemplars in the
     prompt.
@@ -100,6 +100,7 @@ def Bias(model, min_threshold=7):
     response = call_model(
         system_prompt=SYSTEM,
         user_prompt=USER.format(prompt_to_test=model.prompt.template),
+        judge_llm=judge_llm,
     )
 
     score = get_score(response)

validmind/tests/prompt_validation/Clarity.py

@@ -46,7 +46,7 @@ Prompt:
 
 @tags("llm", "zero_shot", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Clarity(model, min_threshold=7):
+def Clarity(model, min_threshold=7, judge_llm=None):
     """
     Evaluates and scores the clarity of prompts in a Large Language Model based on specified guidelines.
 
@@ -89,6 +89,7 @@ def Clarity(model, min_threshold=7):
     response = call_model(
         system_prompt=SYSTEM,
         user_prompt=USER.format(prompt_to_test=model.prompt.template),
+        judge_llm=judge_llm,
     )
 
     score = get_score(response)

validmind/tests/prompt_validation/Conciseness.py

@@ -54,7 +54,7 @@ Prompt:
 
 @tags("llm", "zero_shot", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Conciseness(model, min_threshold=7):
+def Conciseness(model, min_threshold=7, judge_llm=None):
     """
     Analyzes and grades the conciseness of prompts provided to a Large Language Model.
 
@@ -97,6 +97,7 @@ def Conciseness(model, min_threshold=7):
     response = call_model(
         system_prompt=SYSTEM,
         user_prompt=USER.format(prompt_to_test=model.prompt.template),
+        judge_llm=judge_llm,
     )
     score = get_score(response)
     explanation = get_explanation(response)

validmind/tests/prompt_validation/Delimitation.py

@@ -39,7 +39,7 @@ Prompt:
 
 @tags("llm", "zero_shot", "few_shot")
 @tasks("text_classification", "text_summarization")
-def Delimitation(model, min_threshold=7):
+def Delimitation(model, min_threshold=7, judge_llm=None):
     """
     Evaluates the proper use of delimiters in prompts provided to Large Language Models.
 
@@ -83,6 +83,7 @@ def Delimitation(model, min_threshold=7):
     response = call_model(
         system_prompt=SYSTEM,
         user_prompt=USER.format(prompt_to_test=model.prompt.template),
+        judge_llm=judge_llm,
     )
     score = get_score(response)
     explanation = get_explanation(response)