validmind 2.5.24__py3-none-any.whl → 2.6.7__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (198) hide show
  1. validmind/__init__.py +8 -17
  2. validmind/__version__.py +1 -1
  3. validmind/ai/test_descriptions.py +66 -85
  4. validmind/ai/test_result_description/context.py +2 -2
  5. validmind/ai/utils.py +26 -1
  6. validmind/api_client.py +43 -79
  7. validmind/client.py +5 -7
  8. validmind/client_config.py +1 -1
  9. validmind/datasets/__init__.py +1 -1
  10. validmind/datasets/classification/customer_churn.py +7 -5
  11. validmind/datasets/nlp/__init__.py +2 -2
  12. validmind/errors.py +6 -10
  13. validmind/html_templates/content_blocks.py +18 -16
  14. validmind/logging.py +21 -16
  15. validmind/tests/__init__.py +28 -5
  16. validmind/tests/__types__.py +186 -170
  17. validmind/tests/_store.py +7 -21
  18. validmind/tests/comparison.py +362 -0
  19. validmind/tests/data_validation/ACFandPACFPlot.py +44 -73
  20. validmind/tests/data_validation/ADF.py +49 -83
  21. validmind/tests/data_validation/AutoAR.py +59 -96
  22. validmind/tests/data_validation/AutoMA.py +59 -96
  23. validmind/tests/data_validation/AutoStationarity.py +66 -114
  24. validmind/tests/data_validation/ClassImbalance.py +48 -117
  25. validmind/tests/data_validation/DatasetDescription.py +180 -209
  26. validmind/tests/data_validation/DatasetSplit.py +50 -75
  27. validmind/tests/data_validation/DescriptiveStatistics.py +59 -85
  28. validmind/tests/data_validation/{DFGLSArch.py → DickeyFullerGLS.py} +44 -76
  29. validmind/tests/data_validation/Duplicates.py +21 -90
  30. validmind/tests/data_validation/EngleGrangerCoint.py +53 -75
  31. validmind/tests/data_validation/HighCardinality.py +32 -80
  32. validmind/tests/data_validation/HighPearsonCorrelation.py +29 -97
  33. validmind/tests/data_validation/IQROutliersBarPlot.py +63 -94
  34. validmind/tests/data_validation/IQROutliersTable.py +40 -80
  35. validmind/tests/data_validation/IsolationForestOutliers.py +41 -63
  36. validmind/tests/data_validation/KPSS.py +33 -81
  37. validmind/tests/data_validation/LaggedCorrelationHeatmap.py +47 -95
  38. validmind/tests/data_validation/MissingValues.py +17 -58
  39. validmind/tests/data_validation/MissingValuesBarPlot.py +61 -87
  40. validmind/tests/data_validation/PhillipsPerronArch.py +56 -79
  41. validmind/tests/data_validation/RollingStatsPlot.py +50 -81
  42. validmind/tests/data_validation/SeasonalDecompose.py +102 -184
  43. validmind/tests/data_validation/Skewness.py +27 -64
  44. validmind/tests/data_validation/SpreadPlot.py +34 -57
  45. validmind/tests/data_validation/TabularCategoricalBarPlots.py +46 -65
  46. validmind/tests/data_validation/TabularDateTimeHistograms.py +23 -45
  47. validmind/tests/data_validation/TabularNumericalHistograms.py +27 -46
  48. validmind/tests/data_validation/TargetRateBarPlots.py +54 -93
  49. validmind/tests/data_validation/TimeSeriesFrequency.py +48 -133
  50. validmind/tests/data_validation/TimeSeriesHistogram.py +24 -3
  51. validmind/tests/data_validation/TimeSeriesLinePlot.py +29 -47
  52. validmind/tests/data_validation/TimeSeriesMissingValues.py +59 -135
  53. validmind/tests/data_validation/TimeSeriesOutliers.py +54 -171
  54. validmind/tests/data_validation/TooManyZeroValues.py +21 -70
  55. validmind/tests/data_validation/UniqueRows.py +23 -62
  56. validmind/tests/data_validation/WOEBinPlots.py +83 -109
  57. validmind/tests/data_validation/WOEBinTable.py +28 -69
  58. validmind/tests/data_validation/ZivotAndrewsArch.py +33 -75
  59. validmind/tests/data_validation/nlp/CommonWords.py +49 -57
  60. validmind/tests/data_validation/nlp/Hashtags.py +27 -49
  61. validmind/tests/data_validation/nlp/LanguageDetection.py +7 -13
  62. validmind/tests/data_validation/nlp/Mentions.py +32 -63
  63. validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +89 -14
  64. validmind/tests/data_validation/nlp/Punctuations.py +63 -47
  65. validmind/tests/data_validation/nlp/Sentiment.py +4 -0
  66. validmind/tests/data_validation/nlp/StopWords.py +62 -91
  67. validmind/tests/data_validation/nlp/TextDescription.py +116 -159
  68. validmind/tests/data_validation/nlp/Toxicity.py +12 -4
  69. validmind/tests/decorator.py +33 -242
  70. validmind/tests/load.py +212 -153
  71. validmind/tests/model_validation/BertScore.py +13 -7
  72. validmind/tests/model_validation/BleuScore.py +4 -0
  73. validmind/tests/model_validation/ClusterSizeDistribution.py +24 -47
  74. validmind/tests/model_validation/ContextualRecall.py +3 -0
  75. validmind/tests/model_validation/FeaturesAUC.py +43 -74
  76. validmind/tests/model_validation/MeteorScore.py +3 -0
  77. validmind/tests/model_validation/RegardScore.py +5 -1
  78. validmind/tests/model_validation/RegressionResidualsPlot.py +54 -75
  79. validmind/tests/model_validation/embeddings/ClusterDistribution.py +10 -33
  80. validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +11 -29
  81. validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +19 -31
  82. validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +40 -49
  83. validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +29 -15
  84. validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +25 -11
  85. validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +28 -13
  86. validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +67 -38
  87. validmind/tests/model_validation/embeddings/utils.py +53 -0
  88. validmind/tests/model_validation/ragas/AnswerCorrectness.py +37 -32
  89. validmind/tests/model_validation/ragas/{AspectCritique.py → AspectCritic.py} +33 -27
  90. validmind/tests/model_validation/ragas/ContextEntityRecall.py +44 -41
  91. validmind/tests/model_validation/ragas/ContextPrecision.py +40 -35
  92. validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py +133 -0
  93. validmind/tests/model_validation/ragas/ContextRecall.py +40 -35
  94. validmind/tests/model_validation/ragas/Faithfulness.py +42 -30
  95. validmind/tests/model_validation/ragas/NoiseSensitivity.py +59 -35
  96. validmind/tests/model_validation/ragas/{AnswerRelevance.py → ResponseRelevancy.py} +52 -41
  97. validmind/tests/model_validation/ragas/{AnswerSimilarity.py → SemanticSimilarity.py} +39 -34
  98. validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +13 -16
  99. validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +13 -16
  100. validmind/tests/model_validation/sklearn/ClassifierPerformance.py +51 -89
  101. validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +31 -61
  102. validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +118 -83
  103. validmind/tests/model_validation/sklearn/CompletenessScore.py +13 -16
  104. validmind/tests/model_validation/sklearn/ConfusionMatrix.py +62 -94
  105. validmind/tests/model_validation/sklearn/FeatureImportance.py +7 -8
  106. validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +12 -15
  107. validmind/tests/model_validation/sklearn/HomogeneityScore.py +12 -15
  108. validmind/tests/model_validation/sklearn/HyperParametersTuning.py +23 -53
  109. validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +60 -74
  110. validmind/tests/model_validation/sklearn/MinimumAccuracy.py +16 -84
  111. validmind/tests/model_validation/sklearn/MinimumF1Score.py +22 -72
  112. validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +29 -78
  113. validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py +52 -82
  114. validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +51 -145
  115. validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +60 -78
  116. validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +130 -172
  117. validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +26 -55
  118. validmind/tests/model_validation/sklearn/ROCCurve.py +43 -77
  119. validmind/tests/model_validation/sklearn/RegressionPerformance.py +41 -94
  120. validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +47 -136
  121. validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py +164 -208
  122. validmind/tests/model_validation/sklearn/SilhouettePlot.py +54 -99
  123. validmind/tests/model_validation/sklearn/TrainingTestDegradation.py +50 -124
  124. validmind/tests/model_validation/sklearn/VMeasure.py +12 -15
  125. validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +225 -281
  126. validmind/tests/model_validation/statsmodels/AutoARIMA.py +40 -45
  127. validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +22 -47
  128. validmind/tests/model_validation/statsmodels/Lilliefors.py +17 -28
  129. validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +37 -81
  130. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +37 -105
  131. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +62 -166
  132. validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +57 -119
  133. validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +20 -57
  134. validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +47 -80
  135. validmind/tests/ongoing_monitoring/PredictionCorrelation.py +2 -0
  136. validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +4 -2
  137. validmind/tests/output.py +120 -0
  138. validmind/tests/prompt_validation/Bias.py +55 -98
  139. validmind/tests/prompt_validation/Clarity.py +56 -99
  140. validmind/tests/prompt_validation/Conciseness.py +63 -101
  141. validmind/tests/prompt_validation/Delimitation.py +48 -89
  142. validmind/tests/prompt_validation/NegativeInstruction.py +62 -96
  143. validmind/tests/prompt_validation/Robustness.py +80 -121
  144. validmind/tests/prompt_validation/Specificity.py +61 -95
  145. validmind/tests/prompt_validation/ai_powered_test.py +2 -2
  146. validmind/tests/run.py +314 -496
  147. validmind/tests/test_providers.py +109 -79
  148. validmind/tests/utils.py +91 -0
  149. validmind/unit_metrics/__init__.py +16 -155
  150. validmind/unit_metrics/classification/F1.py +1 -0
  151. validmind/unit_metrics/classification/Precision.py +1 -0
  152. validmind/unit_metrics/classification/ROC_AUC.py +1 -0
  153. validmind/unit_metrics/classification/Recall.py +1 -0
  154. validmind/unit_metrics/regression/AdjustedRSquaredScore.py +1 -0
  155. validmind/unit_metrics/regression/GiniCoefficient.py +1 -0
  156. validmind/unit_metrics/regression/HuberLoss.py +1 -0
  157. validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py +1 -0
  158. validmind/unit_metrics/regression/MeanAbsoluteError.py +1 -0
  159. validmind/unit_metrics/regression/MeanAbsolutePercentageError.py +1 -0
  160. validmind/unit_metrics/regression/MeanBiasDeviation.py +1 -0
  161. validmind/unit_metrics/regression/MeanSquaredError.py +1 -0
  162. validmind/unit_metrics/regression/QuantileLoss.py +1 -0
  163. validmind/unit_metrics/regression/RSquaredScore.py +2 -1
  164. validmind/unit_metrics/regression/RootMeanSquaredError.py +1 -0
  165. validmind/utils.py +66 -17
  166. validmind/vm_models/__init__.py +2 -17
  167. validmind/vm_models/dataset/dataset.py +31 -4
  168. validmind/vm_models/figure.py +7 -37
  169. validmind/vm_models/model.py +3 -0
  170. validmind/vm_models/result/__init__.py +7 -0
  171. validmind/vm_models/result/result.jinja +21 -0
  172. validmind/vm_models/result/result.py +337 -0
  173. validmind/vm_models/result/utils.py +160 -0
  174. validmind/vm_models/test_suite/runner.py +16 -54
  175. validmind/vm_models/test_suite/summary.py +3 -3
  176. validmind/vm_models/test_suite/test.py +43 -77
  177. validmind/vm_models/test_suite/test_suite.py +8 -40
  178. validmind-2.6.7.dist-info/METADATA +137 -0
  179. {validmind-2.5.24.dist-info → validmind-2.6.7.dist-info}/RECORD +182 -189
  180. validmind/tests/data_validation/AutoSeasonality.py +0 -190
  181. validmind/tests/metadata.py +0 -59
  182. validmind/tests/model_validation/embeddings/StabilityAnalysis.py +0 -176
  183. validmind/tests/model_validation/ragas/ContextUtilization.py +0 -161
  184. validmind/tests/model_validation/sklearn/ClusterPerformance.py +0 -80
  185. validmind/unit_metrics/composite.py +0 -238
  186. validmind/vm_models/test/metric.py +0 -98
  187. validmind/vm_models/test/metric_result.py +0 -61
  188. validmind/vm_models/test/output_template.py +0 -55
  189. validmind/vm_models/test/result_summary.py +0 -76
  190. validmind/vm_models/test/result_wrapper.py +0 -488
  191. validmind/vm_models/test/test.py +0 -103
  192. validmind/vm_models/test/threshold_test.py +0 -106
  193. validmind/vm_models/test/threshold_test_result.py +0 -75
  194. validmind/vm_models/test_context.py +0 -259
  195. validmind-2.5.24.dist-info/METADATA +0 -118
  196. {validmind-2.5.24.dist-info → validmind-2.6.7.dist-info}/LICENSE +0 -0
  197. {validmind-2.5.24.dist-info → validmind-2.6.7.dist-info}/WHEEL +0 -0
  198. {validmind-2.5.24.dist-info → validmind-2.6.7.dist-info}/entry_points.txt +0 -0
validmind/tests/__types__.py CHANGED
@@ -8,175 +8,191 @@ This module is auto-generated by running `make generate-test-id-types`.
8
8
  Should not be modified manually.
9
9
  """
10
10
 
11
- from typing import Literal
11
+ from typing import Literal, Union
12
12
 
13
- TestID = Literal[
14
- "validmind.prompt_validation.Bias",
15
- "validmind.prompt_validation.Clarity",
16
- "validmind.prompt_validation.Specificity",
17
- "validmind.prompt_validation.Robustness",
18
- "validmind.prompt_validation.NegativeInstruction",
19
- "validmind.prompt_validation.Conciseness",
20
- "validmind.prompt_validation.Delimitation",
21
- "validmind.model_validation.ModelPredictionResiduals",
22
- "validmind.model_validation.BertScore",
23
- "validmind.model_validation.TimeSeriesPredictionsPlot",
24
- "validmind.model_validation.RegardScore",
25
- "validmind.model_validation.BleuScore",
26
- "validmind.model_validation.TimeSeriesPredictionWithCI",
27
- "validmind.model_validation.RegressionResidualsPlot",
28
- "validmind.model_validation.FeaturesAUC",
29
- "validmind.model_validation.ContextualRecall",
30
- "validmind.model_validation.MeteorScore",
31
- "validmind.model_validation.RougeScore",
32
- "validmind.model_validation.ModelMetadata",
33
- "validmind.model_validation.ClusterSizeDistribution",
34
- "validmind.model_validation.TokenDisparity",
35
- "validmind.model_validation.ToxicityScore",
36
- "validmind.model_validation.TimeSeriesR2SquareBySegments",
37
- "validmind.model_validation.embeddings.CosineSimilarityComparison",
38
- "validmind.model_validation.embeddings.EmbeddingsVisualization2D",
39
- "validmind.model_validation.embeddings.StabilityAnalysisRandomNoise",
40
- "validmind.model_validation.embeddings.TSNEComponentsPairwisePlots",
41
- "validmind.model_validation.embeddings.CosineSimilarityDistribution",
42
- "validmind.model_validation.embeddings.PCAComponentsPairwisePlots",
43
- "validmind.model_validation.embeddings.CosineSimilarityHeatmap",
44
- "validmind.model_validation.embeddings.StabilityAnalysisTranslation",
45
- "validmind.model_validation.embeddings.EuclideanDistanceComparison",
46
- "validmind.model_validation.embeddings.ClusterDistribution",
47
- "validmind.model_validation.embeddings.EuclideanDistanceHeatmap",
48
- "validmind.model_validation.embeddings.StabilityAnalysis",
49
- "validmind.model_validation.embeddings.StabilityAnalysisKeyword",
50
- "validmind.model_validation.embeddings.StabilityAnalysisSynonyms",
51
- "validmind.model_validation.embeddings.DescriptiveAnalytics",
52
- "validmind.model_validation.ragas.ContextEntityRecall",
53
- "validmind.model_validation.ragas.Faithfulness",
54
- "validmind.model_validation.ragas.AspectCritique",
55
- "validmind.model_validation.ragas.NoiseSensitivity",
56
- "validmind.model_validation.ragas.AnswerSimilarity",
57
- "validmind.model_validation.ragas.AnswerCorrectness",
58
- "validmind.model_validation.ragas.ContextRecall",
59
- "validmind.model_validation.ragas.ContextPrecision",
60
- "validmind.model_validation.ragas.AnswerRelevance",
61
- "validmind.model_validation.ragas.ContextUtilization",
62
- "validmind.model_validation.sklearn.AdjustedMutualInformation",
63
- "validmind.model_validation.sklearn.SilhouettePlot",
64
- "validmind.model_validation.sklearn.RobustnessDiagnosis",
65
- "validmind.model_validation.sklearn.AdjustedRandIndex",
66
- "validmind.model_validation.sklearn.SHAPGlobalImportance",
67
- "validmind.model_validation.sklearn.ConfusionMatrix",
68
- "validmind.model_validation.sklearn.HomogeneityScore",
69
- "validmind.model_validation.sklearn.CompletenessScore",
70
- "validmind.model_validation.sklearn.OverfitDiagnosis",
71
- "validmind.model_validation.sklearn.ClusterPerformanceMetrics",
72
- "validmind.model_validation.sklearn.PermutationFeatureImportance",
73
- "validmind.model_validation.sklearn.FowlkesMallowsScore",
74
- "validmind.model_validation.sklearn.MinimumROCAUCScore",
75
- "validmind.model_validation.sklearn.ClusterCosineSimilarity",
76
- "validmind.model_validation.sklearn.PrecisionRecallCurve",
77
- "validmind.model_validation.sklearn.ClassifierPerformance",
78
- "validmind.model_validation.sklearn.VMeasure",
79
- "validmind.model_validation.sklearn.MinimumF1Score",
80
- "validmind.model_validation.sklearn.RegressionPerformance",
81
- "validmind.model_validation.sklearn.ROCCurve",
82
- "validmind.model_validation.sklearn.RegressionR2Square",
83
- "validmind.model_validation.sklearn.RegressionErrors",
84
- "validmind.model_validation.sklearn.ClusterPerformance",
85
- "validmind.model_validation.sklearn.TrainingTestDegradation",
86
- "validmind.model_validation.sklearn.RegressionErrorsComparison",
87
- "validmind.model_validation.sklearn.FeatureImportance",
88
- "validmind.model_validation.sklearn.HyperParametersTuning",
89
- "validmind.model_validation.sklearn.KMeansClustersOptimization",
90
- "validmind.model_validation.sklearn.ModelsPerformanceComparison",
91
- "validmind.model_validation.sklearn.WeakspotsDiagnosis",
92
- "validmind.model_validation.sklearn.RegressionR2SquareComparison",
93
- "validmind.model_validation.sklearn.PopulationStabilityIndex",
94
- "validmind.model_validation.sklearn.MinimumAccuracy",
95
- "validmind.model_validation.statsmodels.RegressionModelSensitivityPlot",
96
- "validmind.model_validation.statsmodels.RegressionModelForecastPlotLevels",
97
- "validmind.model_validation.statsmodels.ScorecardHistogram",
98
- "validmind.model_validation.statsmodels.KolmogorovSmirnov",
99
- "validmind.model_validation.statsmodels.CumulativePredictionProbabilities",
100
- "validmind.model_validation.statsmodels.RegressionFeatureSignificance",
101
- "validmind.model_validation.statsmodels.RegressionModelSummary",
102
- "validmind.model_validation.statsmodels.RegressionCoeffs",
103
- "validmind.model_validation.statsmodels.Lilliefors",
104
- "validmind.model_validation.statsmodels.RegressionPermutationFeatureImportance",
105
- "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram",
106
- "validmind.model_validation.statsmodels.AutoARIMA",
107
- "validmind.model_validation.statsmodels.GINITable",
108
- "validmind.model_validation.statsmodels.RegressionModelForecastPlot",
109
- "validmind.model_validation.statsmodels.DurbinWatsonTest",
110
- "validmind.ongoing_monitoring.PredictionCorrelation",
111
- "validmind.ongoing_monitoring.PredictionAcrossEachFeature",
112
- "validmind.ongoing_monitoring.FeatureDrift",
113
- "validmind.ongoing_monitoring.TargetPredictionDistributionPlot",
114
- "validmind.data_validation.IQROutliersTable",
115
- "validmind.data_validation.Skewness",
116
- "validmind.data_validation.BoxPierce",
117
- "validmind.data_validation.Duplicates",
118
- "validmind.data_validation.MissingValuesBarPlot",
119
- "validmind.data_validation.DatasetDescription",
120
- "validmind.data_validation.ProtectedClassesCombination",
121
- "validmind.data_validation.ZivotAndrewsArch",
122
- "validmind.data_validation.ScatterPlot",
123
- "validmind.data_validation.TimeSeriesOutliers",
124
- "validmind.data_validation.TabularCategoricalBarPlots",
125
- "validmind.data_validation.AutoStationarity",
126
- "validmind.data_validation.DescriptiveStatistics",
127
- "validmind.data_validation.TimeSeriesDescription",
128
- "validmind.data_validation.LJungBox",
129
- "validmind.data_validation.TargetRateBarPlots",
130
- "validmind.data_validation.JarqueBera",
131
- "validmind.data_validation.PearsonCorrelationMatrix",
132
- "validmind.data_validation.FeatureTargetCorrelationPlot",
133
- "validmind.data_validation.TabularNumericalHistograms",
134
- "validmind.data_validation.IsolationForestOutliers",
135
- "validmind.data_validation.ChiSquaredFeaturesTable",
136
- "validmind.data_validation.HighCardinality",
137
- "validmind.data_validation.MissingValues",
138
- "validmind.data_validation.PhillipsPerronArch",
139
- "validmind.data_validation.RollingStatsPlot",
140
- "validmind.data_validation.ProtectedClassesDisparity",
141
- "validmind.data_validation.TabularDescriptionTables",
142
- "validmind.data_validation.AutoMA",
143
- "validmind.data_validation.UniqueRows",
144
- "validmind.data_validation.ShapiroWilk",
145
- "validmind.data_validation.TooManyZeroValues",
146
- "validmind.data_validation.HighPearsonCorrelation",
147
- "validmind.data_validation.ACFandPACFPlot",
148
- "validmind.data_validation.WOEBinTable",
149
- "validmind.data_validation.TimeSeriesFrequency",
150
- "validmind.data_validation.DatasetSplit",
151
- "validmind.data_validation.SpreadPlot",
152
- "validmind.data_validation.TimeSeriesLinePlot",
153
- "validmind.data_validation.KPSS",
154
- "validmind.data_validation.AutoSeasonality",
155
- "validmind.data_validation.ProtectedClassesDescription",
156
- "validmind.data_validation.BivariateScatterPlots",
157
- "validmind.data_validation.EngleGrangerCoint",
158
- "validmind.data_validation.TimeSeriesMissingValues",
159
- "validmind.data_validation.TimeSeriesHistogram",
160
- "validmind.data_validation.RunsTest",
161
- "validmind.data_validation.LaggedCorrelationHeatmap",
162
- "validmind.data_validation.SeasonalDecompose",
163
- "validmind.data_validation.WOEBinPlots",
164
- "validmind.data_validation.ClassImbalance",
165
- "validmind.data_validation.IQROutliersBarPlot",
166
- "validmind.data_validation.DFGLSArch",
167
- "validmind.data_validation.TimeSeriesDescriptiveStatistics",
168
- "validmind.data_validation.AutoAR",
169
- "validmind.data_validation.TabularDateTimeHistograms",
170
- "validmind.data_validation.ProtectedClassesThresholdOptimizer",
171
- "validmind.data_validation.ADF",
172
- "validmind.data_validation.nlp.Toxicity",
173
- "validmind.data_validation.nlp.PolarityAndSubjectivity",
174
- "validmind.data_validation.nlp.Punctuations",
175
- "validmind.data_validation.nlp.Sentiment",
176
- "validmind.data_validation.nlp.CommonWords",
177
- "validmind.data_validation.nlp.Hashtags",
178
- "validmind.data_validation.nlp.LanguageDetection",
179
- "validmind.data_validation.nlp.Mentions",
180
- "validmind.data_validation.nlp.TextDescription",
181
- "validmind.data_validation.nlp.StopWords",
13
+ TestID = Union[
14
+ Literal[
15
+ "validmind.data_validation.ACFandPACFPlot",
16
+ "validmind.data_validation.ADF",
17
+ "validmind.data_validation.AutoAR",
18
+ "validmind.data_validation.AutoMA",
19
+ "validmind.data_validation.AutoStationarity",
20
+ "validmind.data_validation.BivariateScatterPlots",
21
+ "validmind.data_validation.BoxPierce",
22
+ "validmind.data_validation.ChiSquaredFeaturesTable",
23
+ "validmind.data_validation.ClassImbalance",
24
+ "validmind.data_validation.DatasetDescription",
25
+ "validmind.data_validation.DatasetSplit",
26
+ "validmind.data_validation.DescriptiveStatistics",
27
+ "validmind.data_validation.DickeyFullerGLS",
28
+ "validmind.data_validation.Duplicates",
29
+ "validmind.data_validation.EngleGrangerCoint",
30
+ "validmind.data_validation.FeatureTargetCorrelationPlot",
31
+ "validmind.data_validation.HighCardinality",
32
+ "validmind.data_validation.HighPearsonCorrelation",
33
+ "validmind.data_validation.IQROutliersBarPlot",
34
+ "validmind.data_validation.IQROutliersTable",
35
+ "validmind.data_validation.IsolationForestOutliers",
36
+ "validmind.data_validation.JarqueBera",
37
+ "validmind.data_validation.KPSS",
38
+ "validmind.data_validation.LJungBox",
39
+ "validmind.data_validation.LaggedCorrelationHeatmap",
40
+ "validmind.data_validation.MissingValues",
41
+ "validmind.data_validation.MissingValuesBarPlot",
42
+ "validmind.data_validation.PearsonCorrelationMatrix",
43
+ "validmind.data_validation.PhillipsPerronArch",
44
+ "validmind.data_validation.ProtectedClassesCombination",
45
+ "validmind.data_validation.ProtectedClassesDescription",
46
+ "validmind.data_validation.ProtectedClassesDisparity",
47
+ "validmind.data_validation.ProtectedClassesThresholdOptimizer",
48
+ "validmind.data_validation.RollingStatsPlot",
49
+ "validmind.data_validation.RunsTest",
50
+ "validmind.data_validation.ScatterPlot",
51
+ "validmind.data_validation.SeasonalDecompose",
52
+ "validmind.data_validation.ShapiroWilk",
53
+ "validmind.data_validation.Skewness",
54
+ "validmind.data_validation.SpreadPlot",
55
+ "validmind.data_validation.TabularCategoricalBarPlots",
56
+ "validmind.data_validation.TabularDateTimeHistograms",
57
+ "validmind.data_validation.TabularDescriptionTables",
58
+ "validmind.data_validation.TabularNumericalHistograms",
59
+ "validmind.data_validation.TargetRateBarPlots",
60
+ "validmind.data_validation.TimeSeriesDescription",
61
+ "validmind.data_validation.TimeSeriesDescriptiveStatistics",
62
+ "validmind.data_validation.TimeSeriesFrequency",
63
+ "validmind.data_validation.TimeSeriesHistogram",
64
+ "validmind.data_validation.TimeSeriesLinePlot",
65
+ "validmind.data_validation.TimeSeriesMissingValues",
66
+ "validmind.data_validation.TimeSeriesOutliers",
67
+ "validmind.data_validation.TooManyZeroValues",
68
+ "validmind.data_validation.UniqueRows",
69
+ "validmind.data_validation.WOEBinPlots",
70
+ "validmind.data_validation.WOEBinTable",
71
+ "validmind.data_validation.ZivotAndrewsArch",
72
+ "validmind.data_validation.nlp.CommonWords",
73
+ "validmind.data_validation.nlp.Hashtags",
74
+ "validmind.data_validation.nlp.LanguageDetection",
75
+ "validmind.data_validation.nlp.Mentions",
76
+ "validmind.data_validation.nlp.PolarityAndSubjectivity",
77
+ "validmind.data_validation.nlp.Punctuations",
78
+ "validmind.data_validation.nlp.Sentiment",
79
+ "validmind.data_validation.nlp.StopWords",
80
+ "validmind.data_validation.nlp.TextDescription",
81
+ "validmind.data_validation.nlp.Toxicity",
82
+ "validmind.model_validation.BertScore",
83
+ "validmind.model_validation.BleuScore",
84
+ "validmind.model_validation.ClusterSizeDistribution",
85
+ "validmind.model_validation.ContextualRecall",
86
+ "validmind.model_validation.FeaturesAUC",
87
+ "validmind.model_validation.MeteorScore",
88
+ "validmind.model_validation.ModelMetadata",
89
+ "validmind.model_validation.ModelPredictionResiduals",
90
+ "validmind.model_validation.RegardScore",
91
+ "validmind.model_validation.RegressionResidualsPlot",
92
+ "validmind.model_validation.RougeScore",
93
+ "validmind.model_validation.TimeSeriesPredictionWithCI",
94
+ "validmind.model_validation.TimeSeriesPredictionsPlot",
95
+ "validmind.model_validation.TimeSeriesR2SquareBySegments",
96
+ "validmind.model_validation.TokenDisparity",
97
+ "validmind.model_validation.ToxicityScore",
98
+ "validmind.model_validation.embeddings.ClusterDistribution",
99
+ "validmind.model_validation.embeddings.CosineSimilarityComparison",
100
+ "validmind.model_validation.embeddings.CosineSimilarityDistribution",
101
+ "validmind.model_validation.embeddings.CosineSimilarityHeatmap",
102
+ "validmind.model_validation.embeddings.DescriptiveAnalytics",
103
+ "validmind.model_validation.embeddings.EmbeddingsVisualization2D",
104
+ "validmind.model_validation.embeddings.EuclideanDistanceComparison",
105
+ "validmind.model_validation.embeddings.EuclideanDistanceHeatmap",
106
+ "validmind.model_validation.embeddings.PCAComponentsPairwisePlots",
107
+ "validmind.model_validation.embeddings.StabilityAnalysisKeyword",
108
+ "validmind.model_validation.embeddings.StabilityAnalysisRandomNoise",
109
+ "validmind.model_validation.embeddings.StabilityAnalysisSynonyms",
110
+ "validmind.model_validation.embeddings.StabilityAnalysisTranslation",
111
+ "validmind.model_validation.embeddings.TSNEComponentsPairwisePlots",
112
+ "validmind.model_validation.ragas.AnswerCorrectness",
113
+ "validmind.model_validation.ragas.AspectCritic",
114
+ "validmind.model_validation.ragas.ContextEntityRecall",
115
+ "validmind.model_validation.ragas.ContextPrecision",
116
+ "validmind.model_validation.ragas.ContextPrecisionWithoutReference",
117
+ "validmind.model_validation.ragas.ContextRecall",
118
+ "validmind.model_validation.ragas.Faithfulness",
119
+ "validmind.model_validation.ragas.NoiseSensitivity",
120
+ "validmind.model_validation.ragas.ResponseRelevancy",
121
+ "validmind.model_validation.ragas.SemanticSimilarity",
122
+ "validmind.model_validation.sklearn.AdjustedMutualInformation",
123
+ "validmind.model_validation.sklearn.AdjustedRandIndex",
124
+ "validmind.model_validation.sklearn.ClassifierPerformance",
125
+ "validmind.model_validation.sklearn.ClusterCosineSimilarity",
126
+ "validmind.model_validation.sklearn.ClusterPerformanceMetrics",
127
+ "validmind.model_validation.sklearn.CompletenessScore",
128
+ "validmind.model_validation.sklearn.ConfusionMatrix",
129
+ "validmind.model_validation.sklearn.FeatureImportance",
130
+ "validmind.model_validation.sklearn.FowlkesMallowsScore",
131
+ "validmind.model_validation.sklearn.HomogeneityScore",
132
+ "validmind.model_validation.sklearn.HyperParametersTuning",
133
+ "validmind.model_validation.sklearn.KMeansClustersOptimization",
134
+ "validmind.model_validation.sklearn.MinimumAccuracy",
135
+ "validmind.model_validation.sklearn.MinimumF1Score",
136
+ "validmind.model_validation.sklearn.MinimumROCAUCScore",
137
+ "validmind.model_validation.sklearn.ModelsPerformanceComparison",
138
+ "validmind.model_validation.sklearn.OverfitDiagnosis",
139
+ "validmind.model_validation.sklearn.PermutationFeatureImportance",
140
+ "validmind.model_validation.sklearn.PopulationStabilityIndex",
141
+ "validmind.model_validation.sklearn.PrecisionRecallCurve",
142
+ "validmind.model_validation.sklearn.ROCCurve",
143
+ "validmind.model_validation.sklearn.RegressionErrors",
144
+ "validmind.model_validation.sklearn.RegressionErrorsComparison",
145
+ "validmind.model_validation.sklearn.RegressionPerformance",
146
+ "validmind.model_validation.sklearn.RegressionR2Square",
147
+ "validmind.model_validation.sklearn.RegressionR2SquareComparison",
148
+ "validmind.model_validation.sklearn.RobustnessDiagnosis",
149
+ "validmind.model_validation.sklearn.SHAPGlobalImportance",
150
+ "validmind.model_validation.sklearn.SilhouettePlot",
151
+ "validmind.model_validation.sklearn.TrainingTestDegradation",
152
+ "validmind.model_validation.sklearn.VMeasure",
153
+ "validmind.model_validation.sklearn.WeakspotsDiagnosis",
154
+ "validmind.model_validation.statsmodels.AutoARIMA",
155
+ "validmind.model_validation.statsmodels.CumulativePredictionProbabilities",
156
+ "validmind.model_validation.statsmodels.DurbinWatsonTest",
157
+ "validmind.model_validation.statsmodels.GINITable",
158
+ "validmind.model_validation.statsmodels.KolmogorovSmirnov",
159
+ "validmind.model_validation.statsmodels.Lilliefors",
160
+ "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram",
161
+ "validmind.model_validation.statsmodels.RegressionCoeffs",
162
+ "validmind.model_validation.statsmodels.RegressionFeatureSignificance",
163
+ "validmind.model_validation.statsmodels.RegressionModelForecastPlot",
164
+ "validmind.model_validation.statsmodels.RegressionModelForecastPlotLevels",
165
+ "validmind.model_validation.statsmodels.RegressionModelSensitivityPlot",
166
+ "validmind.model_validation.statsmodels.RegressionModelSummary",
167
+ "validmind.model_validation.statsmodels.RegressionPermutationFeatureImportance",
168
+ "validmind.model_validation.statsmodels.ScorecardHistogram",
169
+ "validmind.ongoing_monitoring.FeatureDrift",
170
+ "validmind.ongoing_monitoring.PredictionAcrossEachFeature",
171
+ "validmind.ongoing_monitoring.PredictionCorrelation",
172
+ "validmind.ongoing_monitoring.TargetPredictionDistributionPlot",
173
+ "validmind.prompt_validation.Bias",
174
+ "validmind.prompt_validation.Clarity",
175
+ "validmind.prompt_validation.Conciseness",
176
+ "validmind.prompt_validation.Delimitation",
177
+ "validmind.prompt_validation.NegativeInstruction",
178
+ "validmind.prompt_validation.Robustness",
179
+ "validmind.prompt_validation.Specificity",
180
+ "validmind.unit_metrics.classification.Accuracy",
181
+ "validmind.unit_metrics.classification.F1",
182
+ "validmind.unit_metrics.classification.Precision",
183
+ "validmind.unit_metrics.classification.ROC_AUC",
184
+ "validmind.unit_metrics.classification.Recall",
185
+ "validmind.unit_metrics.regression.AdjustedRSquaredScore",
186
+ "validmind.unit_metrics.regression.GiniCoefficient",
187
+ "validmind.unit_metrics.regression.HuberLoss",
188
+ "validmind.unit_metrics.regression.KolmogorovSmirnovStatistic",
189
+ "validmind.unit_metrics.regression.MeanAbsoluteError",
190
+ "validmind.unit_metrics.regression.MeanAbsolutePercentageError",
191
+ "validmind.unit_metrics.regression.MeanBiasDeviation",
192
+ "validmind.unit_metrics.regression.MeanSquaredError",
193
+ "validmind.unit_metrics.regression.QuantileLoss",
194
+ "validmind.unit_metrics.regression.RSquaredScore",
195
+ "validmind.unit_metrics.regression.RootMeanSquaredError",
196
+ ],
197
+ str,
182
198
  ]
validmind/tests/_store.py CHANGED
@@ -5,6 +5,9 @@
5
5
  """Module for storing loaded tests and test providers"""
6
6
 
7
7
 
8
+ from .test_providers import TestProvider, ValidMindTestProvider
9
+
10
+
8
11
  def singleton(cls):
9
12
  """Decorator to make a class a singleton"""
10
13
  instances = {}
@@ -35,7 +38,7 @@ class TestProviderStore:
35
38
  """
36
39
  return namespace in self.test_providers
37
40
 
38
- def get_test_provider(self, namespace: str):
41
+ def get_test_provider(self, namespace: str) -> TestProvider:
39
42
  """Get a test provider by namespace
40
43
 
41
44
  Args:
@@ -61,7 +64,6 @@ class TestStore:
61
64
 
62
65
  def __init__(self):
63
66
  self.tests = {}
64
- self.custom_tests = {}
65
67
 
66
68
  def get_test(self, test_id: str):
67
69
  """Get a test by test ID
@@ -74,29 +76,13 @@ class TestStore:
74
76
  """
75
77
  return self.tests.get(test_id)
76
78
 
77
- def get_custom_test(self, test_id: str):
78
- """Get a custom test by test ID
79
-
80
- Args:
81
- test_id (str): The test ID
82
-
83
- Returns:
84
- object: The test class or function
85
- """
86
- return self.custom_tests.get(test_id)
87
-
88
- def get_test_ids(self) -> list:
89
- """Get all registered test IDs"""
90
- return list(self.tests.keys())
91
-
92
79
  def register_test(self, test_id: str, test: object = None):
93
80
  """Register a test"""
94
81
  self.tests[test_id] = test
95
82
 
96
- def register_custom_test(self, test_id: str, test: object):
97
- """Register a single one-off custom test"""
98
- self.custom_tests[test_id] = test
99
-
100
83
 
101
84
  test_store = TestStore()
102
85
  test_provider_store = TestProviderStore()
86
+
87
+ # setup built-in test providers
88
+ test_provider_store.register_test_provider("validmind", ValidMindTestProvider())