validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

validmind/tests/data_validation/AutoSeasonality.py

@@ -1,190 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-import numpy as np
-import pandas as pd
-from statsmodels.tsa.seasonal import seasonal_decompose
-
-from validmind.logging import get_logger
-from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableMetadata
-
-logger = get_logger(__name__)
-
-
-class AutoSeasonality(Metric):
-    """
-    Automatically identifies and quantifies optimal seasonality in time series data to improve forecasting model
-    performance.
-
-    ### Purpose
-
-    The AutoSeasonality test aims to automatically detect and identify the best seasonal order or period for each
-    variable in a time series dataset. This detection helps to quantify periodic patterns and seasonality that reoccur
-    at fixed intervals in the data. Understanding the seasonality component can drastically improve prediction
-    accuracy, which is especially significant for forecasting-based models.
-
-    ### Test Mechanism
-
-    This test uses the seasonal decomposition method from the Statsmodels Python library. The function takes the
-    'additive' model type for each variable and applies it within the prescribed range of 'min_period' and
-    'max_period'. It decomposes the seasonality for each period in the range and calculates the mean residual error for
-    each period. The seasonal period that results in the minimum residuals is marked as the 'Best Period'. The test
-    results include the 'Best Period', the calculated residual errors, and a determination of 'Seasonality' or 'No
-    Seasonality'.
-
-    ### Signs of High Risk
-
-    - If the optimal seasonal period (or 'Best Period') is consistently at the maximum or minimum limit of the offered
-    range for a majority of variables, it may suggest that the range set does not adequately capture the true seasonal
-    pattern in the series.
-    - A high average 'Residual Error' for the selected 'Best Period' could indicate issues with the model's performance.
-
-    ### Strengths
-
-    - The metric offers an automatic approach to identifying and quantifying the optimal seasonality, providing a
-    robust method for analyzing time series datasets.
-    - It is applicable to multiple variables in a dataset, providing a comprehensive evaluation of each variable's
-    seasonality.
-    - The use of concrete and measurable statistical methods improves the objectivity and reproducibility of the model.
-
-    ### Limitations
-
-    - This AutoSeasonality metric may not be suitable if the time series data exhibits random walk behavior or lacks
-    clear seasonality, as the seasonal decomposition model may not be appropriate.
-    - The defined range for the seasonal period (min_period and max_period) can influence the outcomes. If the actual
-    seasonality period lies outside this range, this method will not be able to identify the true seasonal order.
-    - This metric may not be able to fully interpret complex patterns that go beyond the simple additive model for
-    seasonal decomposition.
-    - The tool may incorrectly infer seasonality if random fluctuations in the data match the predefined seasonal
-    period range.
-    """
-
-    name = "auto_seasonality"
-    required_inputs = ["dataset"]
-    default_params = {"min_period": 1, "max_period": 4}
-    tasks = ["regression"]
-    tags = [
-        "time_series_data",
-        "forecasting",
-        "statistical_test",
-        "statsmodels",
-        "seasonality",
-    ]
-
-    def evaluate_seasonal_periods(self, series, min_period, max_period):
-        seasonal_periods = []
-        residual_errors = []
-
-        for period in range(min_period, max_period + 1):
-            try:
-                sd = seasonal_decompose(series, model="additive", period=period)
-                residual_error = np.abs(sd.resid.dropna()).mean()
-
-                seasonal_periods.append(period)
-                residual_errors.append(residual_error)
-            except Exception as e:
-                logger.error(f"Error evaluating period {period} for series: {e}")
-
-        return seasonal_periods, residual_errors
-
-    def run(self):
-        # Parse input parameters
-        if "min_period" not in self.params:
-            raise ValueError("min_period must be provided in params")
-        min_period = int(self.params["min_period"])
-
-        if "max_period" not in self.params:
-            raise ValueError("max_period must be provided in params")
-        max_period = int(self.params["max_period"])
-
-        df = self.inputs.dataset.df
-
-        # Create an empty DataFrame to store the results
-        summary_auto_seasonality = pd.DataFrame()
-
-        for col_name, col in df.items():
-            series = col.dropna()
-
-            # Evaluate seasonal periods
-            seasonal_periods, residual_errors = self.evaluate_seasonal_periods(
-                series, min_period, max_period
-            )
-
-            for i, period in enumerate(seasonal_periods):
-                decision = "Seasonality" if period > 1 else "No Seasonality"
-                summary_auto_seasonality = pd.concat(
-                    [
-                        summary_auto_seasonality,
-                        pd.DataFrame(
-                            [
-                                {
-                                    "Variable": col_name,
-                                    "Seasonal Period": period,
-                                    "Residual Error": residual_errors[i],
-                                    "Decision": decision,
-                                }
-                            ]
-                        ),
-                    ],
-                    ignore_index=True,
-                )
-
-        # Convert the 'Seasonal Period' column to integer
-        summary_auto_seasonality["Seasonal Period"] = summary_auto_seasonality[
-            "Seasonal Period"
-        ].astype(int)
-
-        # Create a DataFrame to store the best seasonality period for each variable
-        best_seasonality_period = pd.DataFrame()
-
-        for variable in summary_auto_seasonality["Variable"].unique():
-            temp_df = summary_auto_seasonality[
-                summary_auto_seasonality["Variable"] == variable
-            ]
-            best_row = temp_df[
-                temp_df["Residual Error"] == temp_df["Residual Error"].min()
-            ]
-            best_seasonality_period = pd.concat([best_seasonality_period, best_row])
-
-        # Rename the 'Seasonal Period' column to 'Best Period'
-        best_seasonality_period = best_seasonality_period.rename(
-            columns={"Seasonal Period": "Best Period"}
-        )
-
-        # Convert the 'Best Period' column to integer
-        best_seasonality_period["Best Period"] = best_seasonality_period[
-            "Best Period"
-        ].astype(int)
-
-        return self.cache_results(
-            {
-                "auto_seasonality": summary_auto_seasonality.to_dict(orient="records"),
-                "best_seasonality_period": best_seasonality_period.to_dict(
-                    orient="records"
-                ),
-            }
-        )
-
-    def summary(self, metric_value):
-        """
-        Build one table for summarizing the auto seasonality results
-        and another for the best seasonality period results
-        """
-        summary_auto_seasonality = metric_value["auto_seasonality"]
-        best_seasonality_period = metric_value["best_seasonality_period"]
-
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=summary_auto_seasonality,
-                    metadata=ResultTableMetadata(title="Auto Seasonality Results"),
-                ),
-                ResultTable(
-                    data=best_seasonality_period,
-                    metadata=ResultTableMetadata(
-                        title="Best Seasonality Period Results"
-                    ),
-                ),
-            ]
-        )

validmind/tests/metadata.py

@@ -1,59 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-import pandas as pd
-
-from validmind.utils import format_dataframe
-
-from .load import list_tests
-
-
-def list_tags():
-    """
-    List unique tags from all test classes.
-    """
-
-    unique_tags = set()
-
-    for test in list_tests(__as_class=True):
-        unique_tags.update(test.tags)
-
-    return list(unique_tags)
-
-
-def list_tasks_and_tags():
-    """
-    List all task types and their associated tags, with one row per task type and
-    all tags for a task type in one row.
-
-    Returns:
-        pandas.DataFrame: A DataFrame with 'Task Type' and concatenated 'Tags'.
-    """
-    task_tags_dict = {}
-
-    for test in list_tests(__as_class=True):
-        for task in test.tasks:
-            task_tags_dict.setdefault(task, set()).update(test.tags)
-
-    return format_dataframe(
-        pd.DataFrame(
-            [
-                {"Task": task, "Tags": ", ".join(tags)}
-                for task, tags in task_tags_dict.items()
-            ]
-        )
-    )
-
-
-def list_tasks():
-    """
-    List unique tasks from all test classes.
-    """
-
-    unique_tasks = set()
-
-    for test in list_tests(__as_class=True):
-        unique_tasks.update(test.tasks)
-
-    return list(unique_tasks)

validmind/tests/model_validation/embeddings/StabilityAnalysis.py

@@ -1,176 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from abc import abstractmethod
-from typing import List
-
-import numpy as np
-import plotly.express as px
-from sklearn.metrics.pairwise import cosine_similarity
-
-from validmind.logging import get_logger
-from validmind.vm_models import (
-    Figure,
-    ResultSummary,
-    ResultTable,
-    ResultTableMetadata,
-    ThresholdTest,
-    ThresholdTestResult,
-)
-
-logger = get_logger(__name__)
-
-
-class StabilityAnalysis(ThresholdTest):
-    """
-     Assesses the stability of embeddings generated by a model when faced with perturbed input data to ensure robustness
-    and consistency.
-
-    ### Purpose
-
-    The Embedding Stability test evaluates the robustness of the embeddings generated by a model when the input text is
-    perturbed. By comparing the cosine similarities between the original and perturbed embeddings, it gauges the
-    model's ability to maintain consistent semantic representations under slight variations in the input data.
-
-    ### Test Mechanism
-
-    This test works by:
-
-    - Perturbing the original text data.
-    - Generating embeddings for both the original and perturbed datasets using the model.
-    - Calculating the cosine similarities between the original and perturbed embeddings.
-    - Analyzing the distribution of these similarities (mean, min, max, median, and standard deviation).
-    - Determining the test result based on whether the mean similarity exceeds a predefined threshold (default is 0.7).
-
-    ### Signs of High Risk
-
-    - Mean cosine similarity below the threshold (default is 0.7).
-    - Large standard deviation of cosine similarities, indicating inconsistency.
-    - Minimum similarity score significantly lower than expected.
-    - Failure to pass the threshold test based on the mean similarity.
-
-    ### Strengths
-
-    - Provides a quantitative measure of embedding stability.
-    - Helps in identifying weaknesses in the model's ability to handle minor input variations.
-    - Visualization of similarity distributions aids in comprehensive analysis.
-    - Easy to interpret results with clear pass/fail criteria.
-
-    ### Limitations
-
-    - Relies on the chosen perturbation method, which may not cover all possible variations in real-world data.
-    - Thresholds for similarity might need adjustment based on specific application requirements.
-    - Cosine similarity, while useful, may not capture all aspects of semantic stability.
-    """
-
-    required_inputs = ["model", "dataset"]
-    default_params = {
-        "mean_similarity_threshold": 0.7,
-    }
-    tasks = ["feature_extraction"]
-    tags = ["llm", "text_data", "embeddings", "visualization"]
-
-    @abstractmethod
-    def perturb_data(self, data: str) -> str:
-        """Perturb a string of text (overriden by subclasses)"""
-        pass
-
-    def summary(self, results: List[ThresholdTestResult], all_passed: bool):
-        results_table = [
-            {
-                "Mean Similarity": result.values["mean_similarity"],
-                "Min Similarity": result.values["min_similarity"],
-                "Max Similarity": result.values["max_similarity"],
-                "Median Similarity": result.values["median_similarity"],
-                "Std Similarity": result.values["std_similarity"],
-                "Pass/Fail": "Pass" if result.passed else "Fail",
-            }
-            for result in results
-        ]
-        return ResultSummary(
-            results=[
-                ResultTable(
-                    data=results_table,
-                    metadata=ResultTableMetadata(
-                        title="Stability Analysis Results for Embeddings Model"
-                    ),
-                )
-            ]
-        )
-
-    def run(self):
-        # Perturb the test dataset
-        text_column = self.inputs.dataset.text_column
-        original = self.inputs.dataset.df[[text_column]]
-        perturbed = original.copy()
-        perturbed.update(
-            perturbed.select_dtypes(include="object").applymap(self.perturb_data)
-        )
-
-        logger.debug(f"Original data: {original}")
-        logger.debug(f"Perturbed data: {perturbed}")
-
-        # Compute embeddings for the original and perturbed dataset
-        original_embeddings = self.inputs.dataset.y_pred(self.inputs.model)
-        perturbed_embeddings = np.stack(self.inputs.model.predict(perturbed))
-
-        # Compute cosine similarities between original and perturbed embeddings
-        similarities = cosine_similarity(
-            original_embeddings, perturbed_embeddings
-        ).diagonal()
-
-        mean = np.mean(similarities)
-        min = np.min(similarities)
-        max = np.max(similarities)
-        median = np.median(similarities)
-        std = np.std(similarities)
-
-        # Determine if the test passed based on the mean similarity and threshold
-        passed = mean > self.params["mean_similarity_threshold"]
-
-        figures = [
-            px.histogram(
-                x=similarities.flatten(),
-                nbins=100,
-                title="Cosine Similarity Distribution",
-                labels={"x": "Cosine Similarity"},
-            ),
-            px.density_contour(
-                x=similarities.flatten(),
-                nbinsx=100,
-                title="Cosine Similarity Density",
-                labels={"x": "Cosine Similarity"},
-                marginal_x="histogram",
-            ),
-            px.box(
-                x=similarities.flatten(),
-                labels={"x": "Cosine Similarity"},
-                title="Cosine Similarity Box Plot",
-            ),
-        ]
-
-        # For this example, we are not caching the results as done in the reference `run` method
-        return self.cache_results(
-            [
-                ThresholdTestResult(
-                    passed=passed,
-                    values={
-                        "mean_similarity": mean,
-                        "min_similarity": min,
-                        "max_similarity": max,
-                        "median_similarity": median,
-                        "std_similarity": std,
-                    },
-                )
-            ],
-            figures=[
-                Figure(
-                    for_object=self,
-                    key=self.name,
-                    figure=fig,
-                )
-                for fig in figures
-            ],
-            passed=passed,
-        )

validmind/tests/model_validation/ragas/ContextUtilization.py

@@ -1,161 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-import warnings
-
-import plotly.express as px
-from datasets import Dataset
-
-from validmind import tags, tasks
-from validmind.errors import MissingDependencyError
-
-from .utils import get_ragas_config, get_renamed_columns
-
-try:
-    from ragas import evaluate
-    from ragas.metrics import context_utilization
-except ImportError as e:
-    raise MissingDependencyError(
-        "Missing required package `ragas` for ContextUtilization. "
-        "Please run `pip install validmind[llm]` to use LLM tests",
-        required_dependencies=["ragas"],
-        extra="llm",
-    ) from e
-
-
-@tags("ragas", "llm", "retrieval_performance")
-@tasks("text_qa", "text_generation", "text_summarization", "text_classification")
-def ContextUtilization(
-    dataset,
-    question_column: str = "question",
-    contexts_column: str = "contexts",
-    answer_column: str = "answer",
-):  # noqa: B950
-    """
-    Assesses how effectively relevant context chunks are utilized in generating answers by evaluating their ranking
-    within the provided contexts.
-
-    ### Purpose
-
-    The Context Utilization test evaluates whether all of the answer-relevant items present in the contexts are ranked
-    higher within the provided retrieval results. This metric is essential for assessing the performance of models,
-    especially those involved in tasks such as text QA, text generation, text summarization, and text classification.
-
-    ### Test Mechanism
-
-    The test calculates Context Utilization using the formula:
-
-    $$
-    \\text{Context Utilization@K} = \\frac{\\sum_{k=1}^{K} \\left( \\text{Precision@k} \\times v_k \\right)}{\\text{Total number of relevant items in the top } K \\text{ results}}
-    $$
-    $$
-    \\text{Precision@k} = {\\text{true positives@k} \\over  (\\text{true positives@k} + \\text{false positives@k})}
-    $$
-
-    Where $K$ is the total number of chunks in `contexts` and $v_k \\in \\{0, 1\\}$ is the relevance indicator at rank $k$.
-
-
-    This test uses columns for questions, contexts, and answers from the dataset and computes context utilization
-    scores, generating a histogram and box plot for visualization.
-
-    #### Configuring Columns
-
-    This metric requires the following columns in your dataset:
-
-    - `question` (str): The text query that was input into the model.
-    - `contexts` (List[str]): A list of text contexts which are retrieved and which will be evaluated to
-       make sure they contain relevant info in the correct order.
-    - `answer` (str): The llm-generated response for the input `question`.
-
-    If the above data is not in the appropriate column, you can specify different column
-    names for these fields using the parameters `question_column`, `contexts_column`
-    and `ground_truth_column`.
-
-    For example, if your dataset has this data stored in different columns, you can
-    pass the following parameters:
-    ```python
-    {
-        "question_column": "question",
-        "contexts_column": "context_info"
-        "ground_truth_column": "my_ground_truth_col",
-    }
-    ```
-
-    If the data is stored as a dictionary in another column, specify the column and key
-    like this:
-    ```python
-    pred_col = dataset.prediction_column(model)
-    params = {
-        "contexts_column": f"{pred_col}.contexts",
-        "ground_truth_column": "my_ground_truth_col",
-    }
-    ```
-
-    For more complex situations, you can use a function to extract the data:
-    ```python
-    pred_col = dataset.prediction_column(model)
-    params = {
-        "contexts_column": lambda x: [x[pred_col]["context_message"]],
-        "ground_truth_column": "my_ground_truth_col",
-    }
-    ```
-
-    ### Signs of High Risk
-
-    - Very low mean or median context utilization scores, indicating poor usage of retrieved contexts.
-    - High standard deviation, suggesting inconsistent model performance.
-    - Low or minimal max scores, pointing to the model's failure to rank relevant contexts at top positions.
-
-    ### Strengths
-
-    - Quantifies the rank of relevant context chunks in generating responses.
-    - Provides clear visualizations through histograms and box plots for ease of interpretation.
-    - Adapts to different dataset schema by allowing configurable column names.
-
-    ### Limitations
-
-    - Assumes the relevance of context chunks is binary and may not capture nuances of partial relevance.
-    - Requires proper context retrieval to be effective; irrelevant context chunks can skew the results.
-    - Dependent on large sample sizes to provide stable and reliable estimates of utilization performance.
-    """
-    warnings.filterwarnings(
-        "ignore",
-        category=FutureWarning,
-        message="promote has been superseded by promote_options='default'.",
-    )
-
-    required_columns = {
-        "question": question_column,
-        "contexts": contexts_column,
-        "answer": answer_column,
-    }
-
-    df = get_renamed_columns(dataset._df, required_columns)
-
-    result_df = evaluate(
-        Dataset.from_pandas(df), metrics=[context_utilization], **get_ragas_config()
-    ).to_pandas()
-
-    fig_histogram = px.histogram(x=result_df["context_utilization"].to_list(), nbins=10)
-    fig_box = px.box(x=result_df["context_utilization"].to_list())
-
-    return (
-        {
-            # "Scores (will not be uploaded to UI)": result_df[
-            #     ["question", "contexts", "answer", "context_utilization"]
-            # ],
-            "Aggregate Scores": [
-                {
-                    "Mean Score": result_df["context_utilization"].mean(),
-                    "Median Score": result_df["context_utilization"].median(),
-                    "Max Score": result_df["context_utilization"].max(),
-                    "Min Score": result_df["context_utilization"].min(),
-                    "Standard Deviation": result_df["context_utilization"].std(),
-                    "Count": result_df.shape[0],
-                }
-            ],
-        },
-        fig_histogram,
-        fig_box,
-    )

validmind/tests/model_validation/sklearn/ClusterPerformance.py

@@ -1,80 +0,0 @@
-# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
-# See the LICENSE file in the root of this repository for details.
-# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
-
-from dataclasses import dataclass
-
-from validmind.vm_models import Metric
-
-
-@dataclass
-class ClusterPerformance(Metric):
-    """
-    Evaluates and compares a clustering model's performance on training and testing datasets using multiple defined
-    metrics.
-
-    ### Purpose
-
-    The Cluster Performance test evaluates the performance of a clustering model on both the training and testing
-    datasets. It assesses how well the model defines, forms, and distinguishes clusters of data.
-
-    ### Test Mechanism
-
-    The test mechanism involves predicting the clusters of the training and testing datasets using the clustering
-    model. After prediction, performance metrics defined in the `metric_info()` method are calculated against the true
-    labels of the datasets. The results for each metric for both datasets are then collated and returned in a
-    summarized table form listing each metric along with its corresponding train and test values.
-
-    ### Signs of High Risk
-
-    - High discrepancy between the performance metric values on the training and testing datasets.
-    - Low performance metric values on both the training and testing datasets.
-    - Consistent deterioration of performance across different metrics.
-
-    ### Strengths
-
-    - Tests the model's performance on both training and testing datasets, helping to identify overfitting or
-    underfitting.
-    - Allows for the use of a broad range of performance metrics, providing a comprehensive evaluation.
-    - Returns a summarized table, making it easy to compare performance across different metrics and datasets.
-
-    ### Limitations
-
-    - The `metric_info()` method needs to be properly overridden in a subclass and metrics must be manually defined.
-    - The test may not capture the model's performance well if clusters are not well-separated or the model struggles
-    with certain clusters.
-    - Does not consider the computational and time complexity of the model.
-    - Binary comparison (train and test) might not capture performance changes under different circumstances or dataset
-    categories.
-    """
-
-    name = "cluster_performance_metrics"
-    required_inputs = ["model", "dataset"]
-    tasks = ["clustering"]
-    tags = [
-        "sklearn",
-        "model_performance",
-    ]
-
-    def cluster_performance_metrics(self, y_true_train, y_pred_train, metric_info):
-        y_true_train = y_true_train.astype(y_pred_train.dtype).flatten()
-        results = []
-        for metric_name, metric_fcn in metric_info.items():
-            train_value = metric_fcn(list(y_true_train), y_pred_train)
-            results.append({metric_name: train_value})
-        return results
-
-    def metric_info(self):
-        raise NotImplementedError
-
-    def run(self):
-        y_true_train = self.inputs.dataset.y
-        class_pred_train = self.inputs.dataset.y_pred(self.inputs.model)
-        y_true_train = y_true_train.astype(class_pred_train.dtype)
-
-        results = self.cluster_performance_metrics(
-            y_true_train,
-            class_pred_train,
-            self.metric_info(),
-        )
-        return self.cache_results(metric_value=results)