validmind 2.3.1__py3-none-any.whl → 2.3.5__py3-none-any.whl

validmind/__init__.py

@@ -60,7 +60,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .tests.decorator import metric, tags, tasks
+from .tests.decorator import metric, tags, tasks, test
 from .utils import run_async  # noqa: E402
 
 
@@ -112,6 +112,7 @@ __all__ = [  # noqa
     "run_test_suite",
     "tags",
     "tasks",
+    "test",
     "tests",
     "test_suites",
     "vm_models",

validmind/__version__.py

@@ -1 +1 @@
-__version__ = "2.3.1"
+__version__ = "2.3.5"

validmind/datasets/regression/fred_timeseries.py

@@ -0,0 +1,272 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+
+import pandas as pd
+
+current_path = os.path.dirname(os.path.abspath(__file__))
+mortgage30us_path = os.path.join(current_path, "datasets", "fred", "MORTGAGE30US.csv")
+fedfunds_path = os.path.join(current_path, "datasets", "fred", "FEDFUNDS.csv")
+gs10_path = os.path.join(current_path, "datasets", "fred", "GS10.csv")
+unrate_path = os.path.join(current_path, "datasets", "fred", "UNRATE.csv")
+
+target_column = "MORTGAGE30US"
+feature_columns = ["FEDFUNDS", "GS10", "UNRATE"]
+
+
+def get_common_date_range(dfs):
+    start_dates = [df.index.min() for df in dfs]
+    end_dates = [df.index.max() for df in dfs]
+
+    common_start_date = max(start_dates)
+    common_end_date = min(end_dates)
+
+    return common_start_date, common_end_date
+
+
+def align_date_range(dfs, start_date, end_date):
+    return [df.loc[start_date:end_date] for df in dfs]
+
+
+def load_data():
+    mortgage30us = pd.read_csv(
+        mortgage30us_path, parse_dates=["DATE"], index_col="DATE"
+    )
+    fedfunds = pd.read_csv(fedfunds_path, parse_dates=["DATE"], index_col="DATE")
+    gs10 = pd.read_csv(gs10_path, parse_dates=["DATE"], index_col="DATE")
+    unrate = pd.read_csv(unrate_path, parse_dates=["DATE"], index_col="DATE")
+
+    # Resample mortgage30us to monthly frequency
+    mortgage30us = mortgage30us.resample("MS").last()
+
+    # Get the common date range
+    common_start_date, common_end_date = get_common_date_range(
+        [mortgage30us, fedfunds, gs10, unrate]
+    )
+
+    # Align the date range for all dataframes
+    mortgage30us, fedfunds, gs10, unrate = align_date_range(
+        [mortgage30us, fedfunds, gs10, unrate], common_start_date, common_end_date
+    )
+
+    # Combine into a single DataFrame
+    df = pd.concat([mortgage30us, fedfunds, gs10, unrate], axis=1, join="inner")
+    df.columns = [target_column] + feature_columns
+
+    return df
+
+
+# Convert data back to levels
+def convert_to_levels(diff_df, original_df, target_column):
+    """
+    Convert differenced data back to original levels.
+    """
+    previous_values = original_df[target_column].shift(1).dropna()
+    levels_df = diff_df.add(previous_values, axis=0)
+    return levels_df
+
+
+def get_demo_test_config(test_suite=None):
+
+    default_config = {}
+
+    default_config["validmind.data_validation.TimeSeriesDescription"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesLinePlot"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesMissingValues"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.SeasonalDecompose"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TimeSeriesDescriptiveStatistics:train_diff_data"
+    ] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TimeSeriesDescriptiveStatistics:test_diff_data"
+    ] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesOutliers:train_diff_data"] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        },
+        "params": {"zscore_threshold": 4},
+    }
+    default_config["validmind.data_validation.TimeSeriesOutliers:test_diff_data"] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        },
+        "params": {"zscore_threshold": 4},
+    }
+    default_config["validmind.data_validation.TimeSeriesHistogram:train_diff_data"] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        },
+        "params": {"nbins": 100},
+    }
+    default_config["validmind.data_validation.TimeSeriesHistogram:test_diff_data"] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        },
+        "params": {"nbins": 100},
+    }
+    default_config["validmind.data_validation.DatasetSplit"] = {
+        "inputs": {
+            "datasets": ["train_diff_ds", "test_diff_ds"],
+        }
+    }
+    default_config["validmind.model_validation.ModelMetadataComparison"] = {
+        "inputs": {
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionErrorsComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionErrorsComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionR2SquareComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionR2SquareComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesR2SquareBySegments:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesR2SquareBySegments:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        },
+        "params": {
+            "segments": {
+                "start_date": ["2012-11-01", "2018-02-01"],
+                "end_date": ["2018-01-01", "2023-03-01"],
+            }
+        },
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionsPlot:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config["validmind.model_validation.TimeSeriesPredictionsPlot:test_data"] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionWithCI:random_forests_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "random_forests_model",
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionWithCI:gradient_boosting_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "gradient_boosting_model",
+        }
+    }
+    default_config["validmind.model_validation.ModelPredictionResiduals:train_data"] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config["validmind.model_validation.ModelPredictionResiduals:test_data"] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.FeatureImportanceComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.FeatureImportanceComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.PermutationFeatureImportance:random_forests_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "random_forests_model",
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.PermutationFeatureImportance:gradient_boosting_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "gradient_boosting_model",
+        }
+    }
+
+    return default_config

validmind/test_suites/__init__.py

@@ -182,7 +182,6 @@ def describe_suite(test_suite_id: str, verbose=False):
                     "Test Suite Section": "",
                     "Test ID": item,
                     "Test Name": test.__name__,
-                    "Test Type": test.test_type,
                 }
             )
         elif isinstance(item, dict):
@@ -195,7 +194,6 @@ def describe_suite(test_suite_id: str, verbose=False):
                         "Test Suite Section": item["section_id"],
                         "Test ID": test_id,
                         "Test Name": test_id_to_name(test_id),
-                        "Test Type": test.test_type,
                     }
                 )
         else:

validmind/tests/__init__.py

@@ -30,7 +30,9 @@ from ..utils import (
     test_id_to_name,
 )
 from ..vm_models import TestContext, TestInput
-from .decorator import metric, tags, tasks
+from .__types__ import TestID
+from .decorator import tags, tasks
+from .decorator import test as test_decorator
 from .test_providers import LocalTestProvider, TestProvider
 
 logger = get_logger(__name__)
@@ -84,7 +86,6 @@ def _pretty_list_tests(tests, truncate=True):
         {
             "ID": test_id,
             "Name": test_id_to_name(test_id),
-            "Test Type": __test_classes[test_id].test_type,
             "Description": _test_description(__test_classes[test_id], truncate),
             "Required Inputs": __test_classes[test_id].required_inputs,
             "Params": __test_classes[test_id].default_params or {},
@@ -340,7 +341,7 @@ def load_test(test_id: str, reload=False):
         # if its a function, we decorate it and then load the class
         # TODO: simplify this as we move towards all functional metrics
         # "_" is used here so it doesn't conflict with other test ids
-        metric("_")(test)
+        test_decorator("_")(test)
         test = __custom_tests["_"]
 
     test.test_id = f"{test_id}:{result_id}" if result_id else test_id
@@ -348,7 +349,7 @@ def load_test(test_id: str, reload=False):
     return test
 
 
-def describe_test(test_id: str = None, raw: bool = False, show: bool = True):
+def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
     """Get or show details about the test
 
     This function can be used to see test details including the test name, description,
@@ -365,7 +366,6 @@ def describe_test(test_id: str = None, raw: bool = False, show: bool = True):
     details = {
         "ID": test_id,
         "Name": test_id_to_name(test_id),
-        "Test Type": test.test_type,
         "Required Inputs": test.required_inputs,
         "Params": test.default_params or {},
         "Description": inspect.getdoc(test).strip() or "",
@@ -407,7 +407,7 @@ def describe_test(test_id: str = None, raw: bool = False, show: bool = True):
 
 
 def run_test(
-    test_id: str = None,
+    test_id: TestID = None,
     name: str = None,
     unit_metrics: list = None,
     params: dict = None,
@@ -451,7 +451,7 @@ def run_test(
 
     if unit_metrics:
         metric_id_name = "".join(word[0].upper() + word[1:] for word in name.split())
-        test_id = f"validmind.composite_metric.{metric_id_name}"
+        test_id = f"validmind.composite_test.{metric_id_name}"
 
         error, TestClass = load_composite_metric(
             unit_metrics=unit_metrics, metric_name=metric_id_name

validmind/tests/__types__.py

@@ -0,0 +1,180 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Literal types for test IDs.
+
+This module is auto-generated by running `make generate-test-id-types`.
+Should not be modified manually.
+"""
+
+from typing import Literal
+
+TestID = Literal[
+    "validmind.prompt_validation.Bias",
+    "validmind.prompt_validation.Clarity",
+    "validmind.prompt_validation.Specificity",
+    "validmind.prompt_validation.Robustness",
+    "validmind.prompt_validation.NegativeInstruction",
+    "validmind.prompt_validation.Conciseness",
+    "validmind.prompt_validation.Delimitation",
+    "validmind.model_validation.ModelPredictionResiduals",
+    "validmind.model_validation.BertScore",
+    "validmind.model_validation.TimeSeriesPredictionsPlot",
+    "validmind.model_validation.RegardScore",
+    "validmind.model_validation.BleuScore",
+    "validmind.model_validation.TimeSeriesPredictionWithCI",
+    "validmind.model_validation.RegressionResidualsPlot",
+    "validmind.model_validation.FeaturesAUC",
+    "validmind.model_validation.ContextualRecall",
+    "validmind.model_validation.MeteorScore",
+    "validmind.model_validation.RougeScore",
+    "validmind.model_validation.ModelMetadata",
+    "validmind.model_validation.ClusterSizeDistribution",
+    "validmind.model_validation.TokenDisparity",
+    "validmind.model_validation.ToxicityScore",
+    "validmind.model_validation.ModelMetadataComparison",
+    "validmind.model_validation.TimeSeriesR2SquareBySegments",
+    "validmind.model_validation.embeddings.CosineSimilarityComparison",
+    "validmind.model_validation.embeddings.EmbeddingsVisualization2D",
+    "validmind.model_validation.embeddings.StabilityAnalysisRandomNoise",
+    "validmind.model_validation.embeddings.TSNEComponentsPairwisePlots",
+    "validmind.model_validation.embeddings.CosineSimilarityDistribution",
+    "validmind.model_validation.embeddings.PCAComponentsPairwisePlots",
+    "validmind.model_validation.embeddings.CosineSimilarityHeatmap",
+    "validmind.model_validation.embeddings.StabilityAnalysisTranslation",
+    "validmind.model_validation.embeddings.EuclideanDistanceComparison",
+    "validmind.model_validation.embeddings.ClusterDistribution",
+    "validmind.model_validation.embeddings.EuclideanDistanceHeatmap",
+    "validmind.model_validation.embeddings.StabilityAnalysis",
+    "validmind.model_validation.embeddings.StabilityAnalysisKeyword",
+    "validmind.model_validation.embeddings.StabilityAnalysisSynonyms",
+    "validmind.model_validation.embeddings.DescriptiveAnalytics",
+    "validmind.model_validation.ragas.ContextEntityRecall",
+    "validmind.model_validation.ragas.Faithfulness",
+    "validmind.model_validation.ragas.AspectCritique",
+    "validmind.model_validation.ragas.AnswerSimilarity",
+    "validmind.model_validation.ragas.AnswerCorrectness",
+    "validmind.model_validation.ragas.ContextRecall",
+    "validmind.model_validation.ragas.ContextRelevancy",
+    "validmind.model_validation.ragas.ContextPrecision",
+    "validmind.model_validation.ragas.AnswerRelevance",
+    "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",
+    "validmind.model_validation.sklearn.AdjustedMutualInformation",
+    "validmind.model_validation.sklearn.SilhouettePlot",
+    "validmind.model_validation.sklearn.RobustnessDiagnosis",
+    "validmind.model_validation.sklearn.AdjustedRandIndex",
+    "validmind.model_validation.sklearn.SHAPGlobalImportance",
+    "validmind.model_validation.sklearn.ConfusionMatrix",
+    "validmind.model_validation.sklearn.HomogeneityScore",
+    "validmind.model_validation.sklearn.CompletenessScore",
+    "validmind.model_validation.sklearn.OverfitDiagnosis",
+    "validmind.model_validation.sklearn.ClusterPerformanceMetrics",
+    "validmind.model_validation.sklearn.PermutationFeatureImportance",
+    "validmind.model_validation.sklearn.FowlkesMallowsScore",
+    "validmind.model_validation.sklearn.MinimumROCAUCScore",
+    "validmind.model_validation.sklearn.ClusterCosineSimilarity",
+    "validmind.model_validation.sklearn.PrecisionRecallCurve",
+    "validmind.model_validation.sklearn.ClassifierPerformance",
+    "validmind.model_validation.sklearn.VMeasure",
+    "validmind.model_validation.sklearn.MinimumF1Score",
+    "validmind.model_validation.sklearn.ROCCurve",
+    "validmind.model_validation.sklearn.RegressionR2Square",
+    "validmind.model_validation.sklearn.RegressionErrors",
+    "validmind.model_validation.sklearn.ClusterPerformance",
+    "validmind.model_validation.sklearn.FeatureImportanceComparison",
+    "validmind.model_validation.sklearn.TrainingTestDegradation",
+    "validmind.model_validation.sklearn.RegressionErrorsComparison",
+    "validmind.model_validation.sklearn.HyperParametersTuning",
+    "validmind.model_validation.sklearn.KMeansClustersOptimization",
+    "validmind.model_validation.sklearn.ModelsPerformanceComparison",
+    "validmind.model_validation.sklearn.WeakspotsDiagnosis",
+    "validmind.model_validation.sklearn.RegressionR2SquareComparison",
+    "validmind.model_validation.sklearn.PopulationStabilityIndex",
+    "validmind.model_validation.sklearn.MinimumAccuracy",
+    "validmind.model_validation.statsmodels.RegressionModelsCoeffs",
+    "validmind.model_validation.statsmodels.BoxPierce",
+    "validmind.model_validation.statsmodels.RegressionCoeffsPlot",
+    "validmind.model_validation.statsmodels.RegressionModelSensitivityPlot",
+    "validmind.model_validation.statsmodels.RegressionModelForecastPlotLevels",
+    "validmind.model_validation.statsmodels.ScorecardHistogram",
+    "validmind.model_validation.statsmodels.LJungBox",
+    "validmind.model_validation.statsmodels.JarqueBera",
+    "validmind.model_validation.statsmodels.KolmogorovSmirnov",
+    "validmind.model_validation.statsmodels.ShapiroWilk",
+    "validmind.model_validation.statsmodels.CumulativePredictionProbabilities",
+    "validmind.model_validation.statsmodels.RegressionFeatureSignificance",
+    "validmind.model_validation.statsmodels.RegressionModelSummary",
+    "validmind.model_validation.statsmodels.Lilliefors",
+    "validmind.model_validation.statsmodels.RunsTest",
+    "validmind.model_validation.statsmodels.RegressionPermutationFeatureImportance",
+    "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram",
+    "validmind.model_validation.statsmodels.AutoARIMA",
+    "validmind.model_validation.statsmodels.GINITable",
+    "validmind.model_validation.statsmodels.RegressionModelForecastPlot",
+    "validmind.model_validation.statsmodels.DurbinWatsonTest",
+    "validmind.data_validation.MissingValuesRisk",
+    "validmind.data_validation.IQROutliersTable",
+    "validmind.data_validation.BivariateFeaturesBarPlots",
+    "validmind.data_validation.Skewness",
+    "validmind.data_validation.Duplicates",
+    "validmind.data_validation.MissingValuesBarPlot",
+    "validmind.data_validation.DatasetDescription",
+    "validmind.data_validation.ZivotAndrewsArch",
+    "validmind.data_validation.ScatterPlot",
+    "validmind.data_validation.TimeSeriesOutliers",
+    "validmind.data_validation.TabularCategoricalBarPlots",
+    "validmind.data_validation.AutoStationarity",
+    "validmind.data_validation.DescriptiveStatistics",
+    "validmind.data_validation.TimeSeriesDescription",
+    "validmind.data_validation.ANOVAOneWayTable",
+    "validmind.data_validation.TargetRateBarPlots",
+    "validmind.data_validation.PearsonCorrelationMatrix",
+    "validmind.data_validation.FeatureTargetCorrelationPlot",
+    "validmind.data_validation.TabularNumericalHistograms",
+    "validmind.data_validation.IsolationForestOutliers",
+    "validmind.data_validation.ChiSquaredFeaturesTable",
+    "validmind.data_validation.HighCardinality",
+    "validmind.data_validation.MissingValues",
+    "validmind.data_validation.PhillipsPerronArch",
+    "validmind.data_validation.RollingStatsPlot",
+    "validmind.data_validation.TabularDescriptionTables",
+    "validmind.data_validation.AutoMA",
+    "validmind.data_validation.UniqueRows",
+    "validmind.data_validation.TooManyZeroValues",
+    "validmind.data_validation.HighPearsonCorrelation",
+    "validmind.data_validation.ACFandPACFPlot",
+    "validmind.data_validation.BivariateHistograms",
+    "validmind.data_validation.WOEBinTable",
+    "validmind.data_validation.HeatmapFeatureCorrelations",
+    "validmind.data_validation.TimeSeriesFrequency",
+    "validmind.data_validation.DatasetSplit",
+    "validmind.data_validation.SpreadPlot",
+    "validmind.data_validation.TimeSeriesLinePlot",
+    "validmind.data_validation.KPSS",
+    "validmind.data_validation.AutoSeasonality",
+    "validmind.data_validation.BivariateScatterPlots",
+    "validmind.data_validation.EngleGrangerCoint",
+    "validmind.data_validation.TimeSeriesMissingValues",
+    "validmind.data_validation.TimeSeriesHistogram",
+    "validmind.data_validation.LaggedCorrelationHeatmap",
+    "validmind.data_validation.SeasonalDecompose",
+    "validmind.data_validation.WOEBinPlots",
+    "validmind.data_validation.ClassImbalance",
+    "validmind.data_validation.IQROutliersBarPlot",
+    "validmind.data_validation.DFGLSArch",
+    "validmind.data_validation.TimeSeriesDescriptiveStatistics",
+    "validmind.data_validation.AutoAR",
+    "validmind.data_validation.TabularDateTimeHistograms",
+    "validmind.data_validation.ADF",
+    "validmind.data_validation.nlp.Toxicity",
+    "validmind.data_validation.nlp.PolarityAndSubjectivity",
+    "validmind.data_validation.nlp.Punctuations",
+    "validmind.data_validation.nlp.Sentiment",
+    "validmind.data_validation.nlp.CommonWords",
+    "validmind.data_validation.nlp.Hashtags",
+    "validmind.data_validation.nlp.LanguageDetection",
+    "validmind.data_validation.nlp.Mentions",
+    "validmind.data_validation.nlp.TextDescription",
+    "validmind.data_validation.nlp.StopWords",
+]