validmind 2.5.8__py3-none-any.whl → 2.5.18__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. validmind/__version__.py +1 -1
  2. validmind/ai/test_descriptions.py +80 -119
  3. validmind/ai/test_result_description/config.yaml +29 -0
  4. validmind/ai/test_result_description/context.py +73 -0
  5. validmind/ai/test_result_description/image_processing.py +124 -0
  6. validmind/ai/test_result_description/system.jinja +39 -0
  7. validmind/ai/test_result_description/user.jinja +25 -0
  8. validmind/api_client.py +89 -43
  9. validmind/client.py +2 -2
  10. validmind/client_config.py +11 -14
  11. validmind/datasets/credit_risk/__init__.py +1 -0
  12. validmind/datasets/credit_risk/datasets/lending_club_biased.csv.gz +0 -0
  13. validmind/datasets/credit_risk/lending_club_bias.py +142 -0
  14. validmind/datasets/regression/fred_timeseries.py +67 -138
  15. validmind/template.py +1 -0
  16. validmind/test_suites/__init__.py +0 -2
  17. validmind/test_suites/statsmodels_timeseries.py +1 -1
  18. validmind/test_suites/summarization.py +0 -1
  19. validmind/test_suites/time_series.py +0 -43
  20. validmind/tests/__types__.py +14 -15
  21. validmind/tests/data_validation/ACFandPACFPlot.py +15 -13
  22. validmind/tests/data_validation/ADF.py +31 -24
  23. validmind/tests/data_validation/AutoAR.py +9 -9
  24. validmind/tests/data_validation/AutoMA.py +23 -16
  25. validmind/tests/data_validation/AutoSeasonality.py +18 -16
  26. validmind/tests/data_validation/AutoStationarity.py +21 -16
  27. validmind/tests/data_validation/BivariateScatterPlots.py +67 -96
  28. validmind/tests/{model_validation/statsmodels → data_validation}/BoxPierce.py +34 -34
  29. validmind/tests/data_validation/ChiSquaredFeaturesTable.py +85 -124
  30. validmind/tests/data_validation/ClassImbalance.py +15 -12
  31. validmind/tests/data_validation/DFGLSArch.py +19 -13
  32. validmind/tests/data_validation/DatasetDescription.py +17 -11
  33. validmind/tests/data_validation/DatasetSplit.py +7 -5
  34. validmind/tests/data_validation/DescriptiveStatistics.py +28 -21
  35. validmind/tests/data_validation/Duplicates.py +33 -25
  36. validmind/tests/data_validation/EngleGrangerCoint.py +35 -33
  37. validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +59 -71
  38. validmind/tests/data_validation/HighCardinality.py +19 -12
  39. validmind/tests/data_validation/HighPearsonCorrelation.py +27 -22
  40. validmind/tests/data_validation/IQROutliersBarPlot.py +13 -10
  41. validmind/tests/data_validation/IQROutliersTable.py +40 -36
  42. validmind/tests/data_validation/IsolationForestOutliers.py +21 -14
  43. validmind/tests/data_validation/JarqueBera.py +70 -0
  44. validmind/tests/data_validation/KPSS.py +34 -29
  45. validmind/tests/data_validation/LJungBox.py +66 -0
  46. validmind/tests/data_validation/LaggedCorrelationHeatmap.py +22 -15
  47. validmind/tests/data_validation/MissingValues.py +32 -27
  48. validmind/tests/data_validation/MissingValuesBarPlot.py +25 -21
  49. validmind/tests/data_validation/PearsonCorrelationMatrix.py +71 -84
  50. validmind/tests/data_validation/PhillipsPerronArch.py +37 -30
  51. validmind/tests/data_validation/ProtectedClassesCombination.py +197 -0
  52. validmind/tests/data_validation/ProtectedClassesDescription.py +130 -0
  53. validmind/tests/data_validation/ProtectedClassesDisparity.py +133 -0
  54. validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py +172 -0
  55. validmind/tests/data_validation/RollingStatsPlot.py +31 -23
  56. validmind/tests/data_validation/RunsTest.py +72 -0
  57. validmind/tests/data_validation/ScatterPlot.py +63 -78
  58. validmind/tests/data_validation/SeasonalDecompose.py +38 -34
  59. validmind/tests/{model_validation/statsmodels → data_validation}/ShapiroWilk.py +35 -30
  60. validmind/tests/data_validation/Skewness.py +35 -37
  61. validmind/tests/data_validation/SpreadPlot.py +35 -35
  62. validmind/tests/data_validation/TabularCategoricalBarPlots.py +23 -17
  63. validmind/tests/data_validation/TabularDateTimeHistograms.py +21 -13
  64. validmind/tests/data_validation/TabularDescriptionTables.py +51 -16
  65. validmind/tests/data_validation/TabularNumericalHistograms.py +25 -22
  66. validmind/tests/data_validation/TargetRateBarPlots.py +21 -14
  67. validmind/tests/data_validation/TimeSeriesDescription.py +25 -18
  68. validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py +23 -17
  69. validmind/tests/data_validation/TimeSeriesFrequency.py +24 -17
  70. validmind/tests/data_validation/TimeSeriesHistogram.py +33 -32
  71. validmind/tests/data_validation/TimeSeriesLinePlot.py +17 -10
  72. validmind/tests/data_validation/TimeSeriesMissingValues.py +15 -10
  73. validmind/tests/data_validation/TimeSeriesOutliers.py +37 -33
  74. validmind/tests/data_validation/TooManyZeroValues.py +16 -11
  75. validmind/tests/data_validation/UniqueRows.py +11 -6
  76. validmind/tests/data_validation/WOEBinPlots.py +23 -16
  77. validmind/tests/data_validation/WOEBinTable.py +35 -30
  78. validmind/tests/data_validation/ZivotAndrewsArch.py +34 -28
  79. validmind/tests/data_validation/nlp/CommonWords.py +21 -14
  80. validmind/tests/data_validation/nlp/Hashtags.py +42 -40
  81. validmind/tests/data_validation/nlp/LanguageDetection.py +33 -14
  82. validmind/tests/data_validation/nlp/Mentions.py +21 -15
  83. validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +32 -9
  84. validmind/tests/data_validation/nlp/Punctuations.py +24 -20
  85. validmind/tests/data_validation/nlp/Sentiment.py +27 -8
  86. validmind/tests/data_validation/nlp/StopWords.py +26 -19
  87. validmind/tests/data_validation/nlp/TextDescription.py +39 -36
  88. validmind/tests/data_validation/nlp/Toxicity.py +32 -9
  89. validmind/tests/decorator.py +81 -42
  90. validmind/tests/model_validation/BertScore.py +36 -27
  91. validmind/tests/model_validation/BleuScore.py +25 -19
  92. validmind/tests/model_validation/ClusterSizeDistribution.py +38 -34
  93. validmind/tests/model_validation/ContextualRecall.py +38 -13
  94. validmind/tests/model_validation/FeaturesAUC.py +32 -13
  95. validmind/tests/model_validation/MeteorScore.py +46 -33
  96. validmind/tests/model_validation/ModelMetadata.py +32 -64
  97. validmind/tests/model_validation/ModelPredictionResiduals.py +75 -73
  98. validmind/tests/model_validation/RegardScore.py +30 -14
  99. validmind/tests/model_validation/RegressionResidualsPlot.py +10 -5
  100. validmind/tests/model_validation/RougeScore.py +36 -30
  101. validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +30 -14
  102. validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +27 -30
  103. validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +68 -63
  104. validmind/tests/model_validation/TokenDisparity.py +31 -23
  105. validmind/tests/model_validation/ToxicityScore.py +26 -17
  106. validmind/tests/model_validation/embeddings/ClusterDistribution.py +24 -20
  107. validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +30 -27
  108. validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +7 -5
  109. validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +32 -23
  110. validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +7 -5
  111. validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +15 -11
  112. validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +29 -29
  113. validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +34 -25
  114. validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +38 -26
  115. validmind/tests/model_validation/embeddings/StabilityAnalysis.py +40 -1
  116. validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +18 -17
  117. validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +40 -45
  118. validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +17 -19
  119. validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +29 -25
  120. validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +38 -28
  121. validmind/tests/model_validation/ragas/AnswerCorrectness.py +5 -4
  122. validmind/tests/model_validation/ragas/AnswerRelevance.py +5 -4
  123. validmind/tests/model_validation/ragas/AnswerSimilarity.py +5 -4
  124. validmind/tests/model_validation/ragas/AspectCritique.py +12 -6
  125. validmind/tests/model_validation/ragas/ContextEntityRecall.py +9 -8
  126. validmind/tests/model_validation/ragas/ContextPrecision.py +5 -4
  127. validmind/tests/model_validation/ragas/ContextRecall.py +5 -4
  128. validmind/tests/model_validation/ragas/ContextUtilization.py +155 -0
  129. validmind/tests/model_validation/ragas/Faithfulness.py +5 -4
  130. validmind/tests/model_validation/ragas/NoiseSensitivity.py +152 -0
  131. validmind/tests/model_validation/ragas/utils.py +6 -0
  132. validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +19 -12
  133. validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +22 -17
  134. validmind/tests/model_validation/sklearn/ClassifierPerformance.py +27 -25
  135. validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +7 -5
  136. validmind/tests/model_validation/sklearn/ClusterPerformance.py +40 -78
  137. validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +15 -17
  138. validmind/tests/model_validation/sklearn/CompletenessScore.py +17 -11
  139. validmind/tests/model_validation/sklearn/ConfusionMatrix.py +22 -15
  140. validmind/tests/model_validation/sklearn/FeatureImportance.py +95 -0
  141. validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +7 -7
  142. validmind/tests/model_validation/sklearn/HomogeneityScore.py +19 -12
  143. validmind/tests/model_validation/sklearn/HyperParametersTuning.py +35 -30
  144. validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +10 -5
  145. validmind/tests/model_validation/sklearn/MinimumAccuracy.py +32 -32
  146. validmind/tests/model_validation/sklearn/MinimumF1Score.py +23 -23
  147. validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +15 -10
  148. validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py +26 -19
  149. validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +38 -18
  150. validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +32 -26
  151. validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +8 -6
  152. validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +24 -17
  153. validmind/tests/model_validation/sklearn/ROCCurve.py +12 -7
  154. validmind/tests/model_validation/sklearn/RegressionErrors.py +74 -130
  155. validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py +27 -12
  156. validmind/tests/model_validation/sklearn/{RegressionModelsPerformanceComparison.py → RegressionPerformance.py} +18 -20
  157. validmind/tests/model_validation/sklearn/RegressionR2Square.py +55 -94
  158. validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py +32 -13
  159. validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +36 -32
  160. validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py +66 -5
  161. validmind/tests/model_validation/sklearn/SilhouettePlot.py +27 -19
  162. validmind/tests/model_validation/sklearn/TrainingTestDegradation.py +25 -18
  163. validmind/tests/model_validation/sklearn/VMeasure.py +14 -13
  164. validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +7 -5
  165. validmind/tests/model_validation/statsmodels/AutoARIMA.py +24 -18
  166. validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +73 -104
  167. validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +59 -32
  168. validmind/tests/model_validation/statsmodels/GINITable.py +44 -77
  169. validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +33 -34
  170. validmind/tests/model_validation/statsmodels/Lilliefors.py +27 -24
  171. validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +86 -119
  172. validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +100 -0
  173. validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +14 -9
  174. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +17 -13
  175. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +46 -43
  176. validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +38 -36
  177. validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +30 -28
  178. validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +18 -11
  179. validmind/tests/model_validation/statsmodels/ScorecardHistogram.py +75 -107
  180. validmind/tests/ongoing_monitoring/FeatureDrift.py +10 -6
  181. validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +31 -25
  182. validmind/tests/ongoing_monitoring/PredictionCorrelation.py +29 -21
  183. validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +31 -23
  184. validmind/tests/prompt_validation/Bias.py +14 -11
  185. validmind/tests/prompt_validation/Clarity.py +16 -14
  186. validmind/tests/prompt_validation/Conciseness.py +7 -5
  187. validmind/tests/prompt_validation/Delimitation.py +23 -22
  188. validmind/tests/prompt_validation/NegativeInstruction.py +7 -5
  189. validmind/tests/prompt_validation/Robustness.py +12 -10
  190. validmind/tests/prompt_validation/Specificity.py +13 -11
  191. validmind/tests/prompt_validation/ai_powered_test.py +6 -0
  192. validmind/tests/run.py +68 -23
  193. validmind/unit_metrics/__init__.py +81 -144
  194. validmind/unit_metrics/classification/{sklearn/Accuracy.py → Accuracy.py} +1 -1
  195. validmind/unit_metrics/classification/{sklearn/F1.py → F1.py} +1 -1
  196. validmind/unit_metrics/classification/{sklearn/Precision.py → Precision.py} +1 -1
  197. validmind/unit_metrics/classification/{sklearn/ROC_AUC.py → ROC_AUC.py} +1 -2
  198. validmind/unit_metrics/classification/{sklearn/Recall.py → Recall.py} +1 -1
  199. validmind/unit_metrics/regression/{sklearn/AdjustedRSquaredScore.py → AdjustedRSquaredScore.py} +1 -1
  200. validmind/unit_metrics/regression/GiniCoefficient.py +1 -1
  201. validmind/unit_metrics/regression/HuberLoss.py +1 -1
  202. validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py +1 -1
  203. validmind/unit_metrics/regression/{sklearn/MeanAbsoluteError.py → MeanAbsoluteError.py} +1 -1
  204. validmind/unit_metrics/regression/MeanAbsolutePercentageError.py +1 -1
  205. validmind/unit_metrics/regression/MeanBiasDeviation.py +1 -1
  206. validmind/unit_metrics/regression/{sklearn/MeanSquaredError.py → MeanSquaredError.py} +1 -1
  207. validmind/unit_metrics/regression/QuantileLoss.py +1 -1
  208. validmind/unit_metrics/regression/{sklearn/RSquaredScore.py → RSquaredScore.py} +1 -1
  209. validmind/unit_metrics/regression/{sklearn/RootMeanSquaredError.py → RootMeanSquaredError.py} +1 -1
  210. validmind/utils.py +4 -0
  211. validmind/vm_models/dataset/dataset.py +2 -0
  212. validmind/vm_models/figure.py +5 -0
  213. validmind/vm_models/test/metric.py +1 -0
  214. validmind/vm_models/test/result_wrapper.py +143 -158
  215. validmind/vm_models/test/threshold_test.py +1 -0
  216. {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/METADATA +4 -3
  217. validmind-2.5.18.dist-info/RECORD +324 -0
  218. validmind/tests/data_validation/ANOVAOneWayTable.py +0 -138
  219. validmind/tests/data_validation/BivariateFeaturesBarPlots.py +0 -142
  220. validmind/tests/data_validation/BivariateHistograms.py +0 -117
  221. validmind/tests/data_validation/HeatmapFeatureCorrelations.py +0 -124
  222. validmind/tests/data_validation/MissingValuesRisk.py +0 -88
  223. validmind/tests/model_validation/ModelMetadataComparison.py +0 -59
  224. validmind/tests/model_validation/sklearn/FeatureImportanceComparison.py +0 -83
  225. validmind/tests/model_validation/statsmodels/JarqueBera.py +0 -73
  226. validmind/tests/model_validation/statsmodels/LJungBox.py +0 -66
  227. validmind/tests/model_validation/statsmodels/RegressionCoeffsPlot.py +0 -135
  228. validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py +0 -103
  229. validmind/tests/model_validation/statsmodels/RunsTest.py +0 -71
  230. validmind-2.5.8.dist-info/RECORD +0 -318
  231. {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/LICENSE +0 -0
  232. {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/WHEEL +0 -0
  233. {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/entry_points.txt +0 -0
@@ -10,41 +10,43 @@ from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableM
10
10
 
11
11
  class EngleGrangerCoint(Metric):
12
12
  """
13
- Validates co-integration in pairs of time series data using the Engle-Granger test and classifies them as
14
- 'Cointegrated' or 'Not cointegrated'.
15
-
16
- **Purpose**: The intent of this Engle-Granger cointegration test is to explore and quantify the degree of
17
- co-movement between pairs of time series variables in a dataset. This is particularly useful in enhancing the
18
- accuracy of predictive regressions whenever the underlying variables are co-integrated, i.e., they move together
19
- over time.
20
-
21
- **Test Mechanism**: The test first drops any non-applicable values from the input dataset and then iterates over
22
- each pair of variables to apply the Engle-Granger cointegration test. The test generates a 'p' value, which is then
23
- compared against a pre-specified threshold (0.05 by default). The pair is labeled as 'Cointegrated' if the 'p'
24
- value is less than or equal to the threshold or 'Not cointegrated' otherwise. A summary table is returned by the
25
- metric showing cointegration results for each variable pair.
26
-
27
- **Signs of High Risk**:
28
- - A high risk might be indicated if a significant number of variables that were hypothesized to be cointegrated do
29
- not pass the test.
30
- - Another sign of high risk is if a considerable number of 'p' values are close to the threshold. This is a risk
31
- because minor fluctuations in the data can switch the decision between 'Cointegrated' and 'Not cointegrated'.
32
-
33
- **Strengths**:
34
- - The Engle-Granger cointegration test provides an effective way to analyze relationships between time series,
35
- particularly in contexts where it's essential to check if variables are moving together in a statistically
36
- significant manner.
37
- - It is useful in various domains, especially finance or economics. Here, predictive models often hinge on
38
- understanding how different variables move together over time.
39
-
40
- **Limitations**:
41
- - The Engle-Granger cointegration test assumes that the time series are integrated of the same order, which isn't
42
- always true in multivariate time series datasets.
13
+ Assesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test.
14
+
15
+ ### Purpose
16
+
17
+ The intent of this Engle-Granger cointegration test is to explore and quantify the degree of co-movement between
18
+ pairs of time series variables in a dataset. This is particularly useful in enhancing the accuracy of predictive
19
+ regressions whenever the underlying variables are co-integrated, i.e., they move together over time.
20
+
21
+ ### Test Mechanism
22
+
23
+ The test first drops any non-applicable values from the input dataset and then iterates over each pair of variables
24
+ to apply the Engle-Granger cointegration test. The test generates a 'p' value, which is then compared against a
25
+ pre-specified threshold (0.05 by default). The pair is labeled as 'Cointegrated' if the 'p' value is less than or
26
+ equal to the threshold or 'Not cointegrated' otherwise. A summary table is returned by the metric showing
27
+ cointegration results for each variable pair.
28
+
29
+ ### Signs of High Risk
30
+
31
+ - A significant number of hypothesized cointegrated variables do not pass the test.
32
+ - A considerable number of 'p' values are close to the threshold, indicating minor data fluctuations can switch the
33
+ decision between 'Cointegrated' and 'Not cointegrated'.
34
+
35
+ ### Strengths
36
+
37
+ - Provides an effective way to analyze relationships between time series, particularly in contexts where it's
38
+ essential to check if variables move together in a statistically significant manner.
39
+ - Useful in various domains, especially finance or economics, where predictive models often hinge on understanding
40
+ how different variables move together over time.
41
+
42
+ ### Limitations
43
+
44
+ - Assumes that the time series are integrated of the same order, which isn't always true in multivariate time
45
+ series datasets.
43
46
  - The presence of non-stationary characteristics in the series or structural breaks can result in falsely positive
44
47
  or negative cointegration results.
45
- - The test may not perform well for small sample sizes due to lack of statistical power. Therefore, it should be
46
- used with caution, and whenever possible, supplemented with other predictive indicators for a more robust model
47
- evaluation.
48
+ - May not perform well for small sample sizes due to lack of statistical power and should be supplemented with
49
+ other predictive indicators for a more robust model evaluation.
48
50
  """
49
51
 
50
52
  type = "dataset"
@@ -2,108 +2,96 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
- from dataclasses import dataclass
6
5
 
7
6
  import numpy as np
8
7
  import plotly.graph_objects as go
9
8
 
10
- from validmind.vm_models import Figure, Metric
9
+ from validmind import tags, tasks
11
10
 
12
11
 
13
- @dataclass
14
- class FeatureTargetCorrelationPlot(Metric):
12
+ @tags("tabular_data", "visualization", "correlation")
13
+ @tasks("classification", "regression")
14
+ def FeatureTargetCorrelationPlot(dataset, fig_height=600):
15
15
  """
16
- Visualizes the correlation between input features and model's target output in a color-coded horizontal bar plot.
16
+ Visualizes the correlation between input features and the model's target output in a color-coded horizontal bar
17
+ plot.
17
18
 
18
- **Purpose**: This test is designed to graphically illustrate the correlations between distinct input features and
19
- the target output of a Machine Learning model. Understanding how each feature influences the model's predictions is
20
- crucial - a higher correlation indicates stronger influence of the feature on the target variable. This correlation
21
- study is especially advantageous during feature selection and for comprehending the model's operation.
19
+ ### Purpose
22
20
 
23
- **Test Mechanism**: This FeatureTargetCorrelationPlot test computes and presents the correlations between the
24
- features and the target variable using a specific dataset. These correlations are calculated, graphically
25
- represented in a horizontal bar plot, and color-coded based on the strength of the correlation. A hovering template
26
- can also be utilized for informative tooltips. It is possible to specify the features to be analyzed and adjust the
27
- graph's height according to need.
21
+ This test is designed to graphically illustrate the correlations between distinct input features and the target
22
+ output of a Machine Learning model. Understanding how each feature influences the model's predictions is crucial—a
23
+ higher correlation indicates a stronger influence of the feature on the target variable. This correlation study is
24
+ especially advantageous during feature selection and for comprehending the model's operation.
25
+
26
+ ### Test Mechanism
27
+
28
+ This FeatureTargetCorrelationPlot test computes and presents the correlations between the features and the target
29
+ variable using a specific dataset. These correlations are calculated and are then graphically represented in a
30
+ horizontal bar plot, color-coded based on the strength of the correlation. A hovering template can also be utilized
31
+ for informative tooltips. It is possible to specify the features to be analyzed and adjust the graph's height
32
+ according to need.
33
+
34
+ ### Signs of High Risk
28
35
 
29
- **Signs of High Risk**:
30
36
  - There are no strong correlations (either positive or negative) between features and the target variable. This
31
37
  could suggest high risk as the supplied features do not appear to significantly impact the prediction output.
32
38
  - The presence of duplicated correlation values might hint at redundancy in the feature set.
33
39
 
34
- **Strengths**:
40
+ ### Strengths
41
+
35
42
  - Provides visual assistance to interpreting correlations more effectively.
36
43
  - Gives a clear and simple tour of how each feature affects the model's target variable.
37
44
  - Beneficial for feature selection and grasping the model's prediction nature.
38
45
  - Precise correlation values for each feature are offered by the hover template, contributing to a granular-level
39
46
  comprehension.
40
47
 
41
- **Limitations**:
48
+ ### Limitations
49
+
42
50
  - The test only accepts numerical data, meaning variables of other types need to be prepared beforehand.
43
51
  - The plot assumes all correlations to be linear, thus non-linear relationships might not be captured effectively.
44
52
  - Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test
45
53
  may not accurately reflect their importance.
46
54
  """
47
55
 
48
- name = "feature_target_correlation_plot"
49
- required_inputs = ["dataset"]
50
- default_params = {"features": None, "fig_height": 600}
51
- tasks = ["classification", "regression"]
52
- tags = ["tabular_data", "visualization", "feature_importance", "correlation"]
56
+ # Filter DataFrame based on features and target_column
57
+ df = dataset.df[dataset.feature_columns + [dataset.target_column]]
53
58
 
54
- def run(self):
55
- fig_height = self.params["fig_height"]
59
+ fig = _visualize_feature_target_correlation(df, dataset.target_column, fig_height)
56
60
 
57
- if self.params["features"] is None:
58
- features = self.inputs.dataset.feature_columns
59
- else:
60
- features = self.params["features"]
61
+ return fig
61
62
 
62
- target_column = self.inputs.dataset.target_column
63
63
 
64
- # Filter DataFrame based on features and target_column
65
- df = self.inputs.dataset.df[features + [target_column]]
64
+ def _visualize_feature_target_correlation(df, target_column, fig_height):
65
+ # Compute correlations with the target variable
66
+ correlations = (
67
+ df.corr(numeric_only=True)[target_column].drop(target_column).to_frame()
68
+ )
69
+ correlations = correlations.loc[:, ~correlations.columns.duplicated()]
66
70
 
67
- figure = self.visualize_feature_target_correlation(
68
- df, target_column, fig_height
69
- )
71
+ correlations = correlations.sort_values(by=target_column, ascending=True)
70
72
 
71
- return self.cache_results(figures=figure)
73
+ # Create a gradual color map from red (1) to blue (-1)
74
+ color_map = np.linspace(1, -1, len(correlations))
75
+ colors = [
76
+ f"rgb({int(255 * (1 - val))}, 0, {int(255 * (1 + val))})" for val in color_map
77
+ ]
72
78
 
73
- def visualize_feature_target_correlation(self, df, target_column, fig_height):
74
- # Compute correlations with the target variable
75
- correlations = (
76
- df.corr(numeric_only=True)[target_column].drop(target_column).to_frame()
77
- )
78
- correlations = correlations.loc[:, ~correlations.columns.duplicated()]
79
-
80
- correlations = correlations.sort_values(by=target_column, ascending=True)
81
-
82
- # Create a gradual color map from red (1) to blue (-1)
83
- color_map = np.linspace(1, -1, len(correlations))
84
- colors = [
85
- f"rgb({int(255 * (1 - val))}, 0, {int(255 * (1 + val))})"
86
- for val in color_map
87
- ]
88
-
89
- # Create a horizontal bar plot with gradual color mapping
90
- fig = go.Figure(
91
- data=go.Bar(
92
- x=correlations[target_column],
93
- y=correlations.index,
94
- orientation="h",
95
- marker=dict(color=colors),
96
- hovertemplate="Feature: %{y}<br>Correlation: %{x:.2f}<extra></extra>", # Hover template for tooltips
97
- )
79
+ # Create a horizontal bar plot with gradual color mapping
80
+ fig = go.Figure(
81
+ data=go.Bar(
82
+ x=correlations[target_column],
83
+ y=correlations.index,
84
+ orientation="h",
85
+ marker=dict(color=colors),
86
+ hovertemplate="Feature: %{y}<br>Correlation: %{x:.2f}<extra></extra>", # Hover template for tooltips
98
87
  )
99
-
100
- # Set the title and axis labels
101
- fig.update_layout(
102
- title=f"Correlations of Features vs Target Variable ({target_column})",
103
- xaxis_title="",
104
- yaxis_title="",
105
- height=fig_height, # Adjust the height value as needed
106
- )
107
-
108
- figure = Figure(for_object=self, key=self.key, figure=fig)
109
- return [figure]
88
+ )
89
+
90
+ # Set the title and axis labels
91
+ fig.update_layout(
92
+ title=f"Correlations of Features vs Target Variable ({target_column})",
93
+ xaxis_title="",
94
+ yaxis_title="",
95
+ height=fig_height, # Adjust the height value as needed
96
+ )
97
+ return fig
@@ -22,26 +22,33 @@ class HighCardinality(ThresholdTest):
22
22
  """
23
23
  Assesses the number of unique values in categorical columns to detect high cardinality and potential overfitting.
24
24
 
25
- **Purpose**: The “High Cardinality” test is used to evaluate the number of unique values present in the categorical
26
- columns of a dataset. In this context, high cardinality implies the presence of a large number of unique,
27
- non-repetitive values in the dataset.
25
+ ### Purpose
28
26
 
29
- **Test Mechanism**: The test first infers the dataset's type and then calculates an initial numeric threshold based
30
- on the test parameters. It only considers columns classified as "Categorical". For each of these columns, the
31
- number of distinct values (n_distinct) and the percentage of distinct values (p_distinct) are calculated. The test
32
- will pass if n_distinct is less than the calculated numeric threshold. Lastly, the results, which include details
33
- such as column name, number of distinct values, and pass/fail status, are compiled into a table.
27
+ The “High Cardinality” test is used to evaluate the number of unique values present in the categorical columns of a
28
+ dataset. In this context, high cardinality implies the presence of a large number of unique, non-repetitive values
29
+ in the dataset.
30
+
31
+ ### Test Mechanism
32
+
33
+ The test first infers the dataset's type and then calculates an initial numeric threshold based on the test
34
+ parameters. It only considers columns classified as "Categorical". For each of these columns, the number of
35
+ distinct values (n_distinct) and the percentage of distinct values (p_distinct) are calculated. The test will pass
36
+ if n_distinct is less than the calculated numeric threshold. Lastly, the results, which include details such as
37
+ column name, number of distinct values, and pass/fail status, are compiled into a table.
38
+
39
+ ### Signs of High Risk
34
40
 
35
- **Signs of High Risk**:
36
41
  - A large number of distinct values (high cardinality) in one or more categorical columns implies a high risk.
37
42
  - A column failing the test (n_distinct >= num_threshold) is another indicator of high risk.
38
43
 
39
- **Strengths**:
44
+ ### Strengths
45
+
40
46
  - The High Cardinality test is effective in early detection of potential overfitting and unwanted noise.
41
47
  - It aids in identifying potential outliers and inconsistencies, thereby improving data quality.
42
- - The test can be applied to both, classification and regression task types, demonstrating its versatility.
48
+ - The test can be applied to both classification and regression task types, demonstrating its versatility.
49
+
50
+ ### Limitations
43
51
 
44
- **Limitations**:
45
52
  - The test is restricted to only "Categorical" data types and is thus not suitable for numerical or continuous
46
53
  features, limiting its scope.
47
54
  - The test does not consider the relevance or importance of unique values in categorical features, potentially
@@ -22,36 +22,41 @@ class HighPearsonCorrelation(ThresholdTest):
22
22
  """
23
23
  Identifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity.
24
24
 
25
- **Purpose**: The High Pearson Correlation test measures the linear relationship between features in a dataset, with
26
- the main goal of identifying high correlations that might indicate feature redundancy or multicollinearity.
27
- Identification of such issue allows developers and risk management teams to properly deal with potential impacts on
28
- the machine learning model's performance and interpretability.
25
+ ### Purpose
29
26
 
30
- **Test Mechanism**: The test works by generating pairwise Pearson correlations for all features in the dataset,
31
- then sorting and eliminating duplicate and self-correlations. It assigns a Pass or Fail based on whether the
32
- absolute value of the correlation coefficient surpasses a pre-set threshold (defaulted at 0.3). It lastly returns
33
- the top ten strongest correlations regardless of passing or failing status.
27
+ The High Pearson Correlation test measures the linear relationship between features in a dataset, with the main
28
+ goal of identifying high correlations that might indicate feature redundancy or multicollinearity. Identification
29
+ of such issues allows developers and risk management teams to properly deal with potential impacts on the machine
30
+ learning model's performance and interpretability.
31
+
32
+ ### Test Mechanism
33
+
34
+ The test works by generating pairwise Pearson correlations for all features in the dataset, then sorting and
35
+ eliminating duplicate and self-correlations. It assigns a Pass or Fail based on whether the absolute value of the
36
+ correlation coefficient surpasses a pre-set threshold (defaulted at 0.3). It lastly returns the top ten strongest
37
+ correlations regardless of passing or failing status.
38
+
39
+ ### Signs of High Risk
34
40
 
35
- **Signs of High Risk**:
36
41
  - A high risk indication would be the presence of correlation coefficients exceeding the threshold.
37
42
  - If the features share a strong linear relationship, this could lead to potential multicollinearity and model
38
43
  overfitting.
39
44
  - Redundancy of variables can undermine the interpretability of the model due to uncertainty over the authenticity
40
45
  of individual variable's predictive power.
41
46
 
42
- **Strengths**:
43
- - The High Pearson Correlation test provides a quick and simple means of identifying relationships between feature
44
- pairs.
45
- - It generates a transparent output which not only displays pairs of correlated variables but also delivers the
46
- Pearson correlation coefficient and a Pass or Fail status for each.
47
- - It aids early identification of potential multicollinearity issues that may disrupt model training.
48
-
49
- **Limitations**:
50
- - The Pearson correlation test can only delineate linear relationships. It fails to shed light on nonlinear
51
- relationships or dependencies.
52
- - It is sensitive to outliers where a few outliers could notably affect the correlation coefficient.
53
- - It is limited to identifying redundancy only within feature pairs. When three or more variables are linearly
54
- dependent, it may fail to spot this complex relationship.
47
+ ### Strengths
48
+
49
+ - Provides a quick and simple means of identifying relationships between feature pairs.
50
+ - Generates a transparent output that displays pairs of correlated variables, the Pearson correlation coefficient,
51
+ and a Pass or Fail status for each.
52
+ - Aids in early identification of potential multicollinearity issues that may disrupt model training.
53
+
54
+ ### Limitations
55
+
56
+ - Can only delineate linear relationships, failing to shed light on nonlinear relationships or dependencies.
57
+ - Sensitive to outliers where a few outliers could notably affect the correlation coefficient.
58
+ - Limited to identifying redundancy only within feature pairs; may fail to spot more complex relationships among
59
+ three or more variables.
55
60
  - The top 10 result filter might not fully capture the richness of the data; an option to configure the number of
56
61
  retained results could be helpful.
57
62
  """
@@ -12,14 +12,18 @@ from validmind.vm_models import Figure, Metric
12
12
  @dataclass
13
13
  class IQROutliersBarPlot(Metric):
14
14
  """
15
- Visualizes outlier distribution across percentiles in numerical data using Interquartile Range (IQR) method.
15
+ Visualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method.
16
16
 
17
- **Purpose**: The InterQuartile Range Outliers Bar Plot (IQROutliersBarPlot) metric aims to visually analyze and
18
- evaluate the extent of outliers in numeric variables based on percentiles. Its primary purpose is to clarify the
19
- dataset's distribution, flag possible abnormalities in it and gauge potential risks associated with processing
20
- potentially skewed data, which can affect the machine learning model's predictive prowess.
17
+ ### Purpose
21
18
 
22
- **Test Mechanism**: The examination invokes a series of steps:
19
+ The InterQuartile Range Outliers Bar Plot (IQROutliersBarPlot) metric aims to visually analyze and evaluate the
20
+ extent of outliers in numeric variables based on percentiles. Its primary purpose is to clarify the dataset's
21
+ distribution, flag possible abnormalities in it, and gauge potential risks associated with processing potentially
22
+ skewed data, which can affect the machine learning model's predictive prowess.
23
+
24
+ ### Test Mechanism
25
+
26
+ The examination invokes a series of steps:
23
27
 
24
28
  1. For every numeric feature in the dataset, the 25th percentile (Q1) and 75th percentile (Q3) are calculated
25
29
  before deriving the Interquartile Range (IQR), the difference between Q1 and Q3.
@@ -31,8 +35,7 @@ class IQROutliersBarPlot(Metric):
31
35
  5. These counts are employed to construct a bar plot for the feature, showcasing the distribution of outliers
32
36
  across different percentiles.
33
37
 
34
- **Signs of High Risk**: High risk or a potential lapse in the model's performance could be unveiled by the
35
- following signs:
38
+ ### Signs of High Risk
36
39
 
37
40
  - A prevalence of outliers in the data, potentially skewing its distribution.
38
41
  - Outliers dominating higher percentiles (75-100) which implies the presence of extreme values, capable of severely
@@ -40,7 +43,7 @@ class IQROutliersBarPlot(Metric):
40
43
  - Certain features harboring most of their values as outliers, which signifies that these features might not
41
44
  contribute positively to the model's forecasting ability.
42
45
 
43
- **Strengths**:
46
+ ### Strengths
44
47
 
45
48
  - Effectively identifies outliers in the data through visual means, facilitating easier comprehension and offering
46
49
  insights into the outliers' possible impact on the model.
@@ -48,7 +51,7 @@ class IQROutliersBarPlot(Metric):
48
51
  - Task-agnostic in nature; it is viable for both classification and regression tasks.
49
52
  - Can handle large datasets as its operation does not hinge on computationally heavy operations.
50
53
 
51
- **Limitations**:
54
+ ### Limitations
52
55
 
53
56
  - Its application is limited to numerical variables and does not extend to categorical ones.
54
57
  - Relies on a predefined threshold (default being 1.5) for outlier identification, which may not be suitable for
@@ -12,42 +12,46 @@ from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableM
12
12
  @dataclass
13
13
  class IQROutliersTable(Metric):
14
14
  """
15
- Determines and summarizes outliers in numerical features using Interquartile Range method.
16
-
17
- **Purpose**: The "Interquartile Range Outliers Table" (IQROutliersTable) metric has been designed for identifying
18
- and summarizing outliers within numerical features of a dataset using the Interquartile Range (IQR) method. The
19
- purpose of this exercise is crucial in the pre-processing of data as outliers can substantially distort the
20
- statistical analysis and debilitate the performance of machine learning models.
21
-
22
- **Test Mechanism**: The IQR, which is the range separating the first quartile (25th percentile) from the third
23
- quartile (75th percentile), is calculated for each numerical feature within the dataset. An outlier is defined as a
24
- data point falling below the "Q1 - 1.5 * IQR" or above "Q3 + 1.5 * IQR" range. The metric then computes the number
25
- of outliers along with their minimum, 25th percentile, median, 75th percentile, and maximum values for each
26
- numerical feature. If no specific features are chosen, the metric will apply to all numerical features in the
27
- dataset. The default outlier threshold is set to 1.5, following the standard definition of outliers in statistical
28
- analysis, although it can be customized by the user.
29
-
30
- **Signs of High Risk**:
31
- - High risk is indicated by a large number of outliers in multiple features.
32
- - Outliers that are significantly distanced from the mean value of variables could potentially signal high risk.
33
- - Data entry errors or other data quality issues could be manifested through extremely high or low outlier values.
34
-
35
- **Strengths**:
36
- - It yields a comprehensive summary of outliers for each numerical feature within the dataset. This enables the
37
- user to pinpoint features with potential quality issues.
38
- - The IQR method is not overly affected by extremely high or low outlier values as it is based on quartile
39
- calculations.
40
- - The versatility of this metric grants the ability to customize the method to work on selected features and set a
41
- defined threshold for outliers.
42
-
43
- **Limitations**:
44
- - The metric might cause false positives if the variable of interest veers away from a normal or near-normal
45
- distribution, notably in the case of skewed distributions.
46
- - It does not extend to provide interpretation or recommendations for tackling outliers and relies on the user or a
47
- data scientist to conduct further analysis of the results.
48
- - As it only functions on numerical features, it cannot be used for categorical data.
49
- - For data that has undergone heavy pre-processing, was manipulated, or inherently possesses a high kurtosis (heavy
50
- tails), the pre-set threshold may not be optimal for outlier detection.
15
+ Determines and summarizes outliers in numerical features using the Interquartile Range method.
16
+
17
+ ### Purpose
18
+
19
+ The "Interquartile Range Outliers Table" (IQROutliersTable) metric is designed to identify and summarize outliers
20
+ within numerical features of a dataset using the Interquartile Range (IQR) method. This exercise is crucial in the
21
+ pre-processing of data because outliers can substantially distort statistical analysis and impact the performance
22
+ of machine learning models.
23
+
24
+ ### Test Mechanism
25
+
26
+ The IQR, which is the range separating the first quartile (25th percentile) from the third quartile (75th
27
+ percentile), is calculated for each numerical feature within the dataset. An outlier is defined as a data point
28
+ falling below the "Q1 - 1.5 * IQR" or above "Q3 + 1.5 * IQR" range. The test computes the number of outliers and
29
+ their summary statistics (minimum, 25th percentile, median, 75th percentile, and maximum values) for each numerical
30
+ feature. If no specific features are chosen, the test applies to all numerical features in the dataset. The default
31
+ outlier threshold is set to 1.5 but can be customized by the user.
32
+
33
+ ### Signs of High Risk
34
+
35
+ - A large number of outliers in multiple features.
36
+ - Outliers significantly distanced from the mean value of variables.
37
+ - Extremely high or low outlier values indicative of data entry errors or other data quality issues.
38
+
39
+ ### Strengths
40
+
41
+ - Provides a comprehensive summary of outliers for each numerical feature, helping pinpoint features with potential
42
+ quality issues.
43
+ - The IQR method is robust to extremely high or low outlier values as it is based on quartile calculations.
44
+ - Can be customized to work on selected features and set thresholds for outliers.
45
+
46
+ ### Limitations
47
+
48
+ - Might cause false positives if the variable deviates from a normal or near-normal distribution, especially for
49
+ skewed distributions.
50
+ - Does not provide interpretation or recommendations for addressing outliers, relying on further analysis by users
51
+ or data scientists.
52
+ - Only applicable to numerical features, not categorical data.
53
+ - Default thresholds may not be optimal for data with heavy pre-processing, manipulation, or inherently high
54
+ kurtosis (heavy tails).
51
55
  """
52
56
 
53
57
  name = "iqr_outliers_table"
@@ -17,24 +17,30 @@ class IsolationForestOutliers(Metric):
17
17
  """
18
18
  Detects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots.
19
19
 
20
- **Purpose**: The `IsolationForestOutliers` test is designed to identify anomalies or outliers in the model's
21
- dataset using the isolation forest algorithm. This algorithm assumes that anomalous data points can be isolated
22
- more quickly due to their distinctive properties. By creating isolation trees and identifying instances with
23
- shorter average path lengths, the test is able to pick out data points that differ from the majority.
24
-
25
- **Test Mechanism**: The test uses the isolation forest algorithm, which builds an ensemble of isolation trees by
26
- randomly selecting features and splitting the data based on random thresholds. It isolates anomalies rather than
27
- focusing on normal data points. For each pair of variables, a scatter plot is generated which distinguishes the
28
- identified outliers from the inliers. The results of the test can be visualized using these scatter plots,
29
- illustrating the distinction between outliers and inliers.
30
-
31
- **Signs of High Risk**:
20
+ ### Purpose
21
+
22
+ The IsolationForestOutliers test is designed to identify anomalies or outliers in the model's dataset using the
23
+ isolation forest algorithm. This algorithm assumes that anomalous data points can be isolated more quickly due to
24
+ their distinctive properties. By creating isolation trees and identifying instances with shorter average path
25
+ lengths, the test is able to pick out data points that differ from the majority.
26
+
27
+ ### Test Mechanism
28
+
29
+ The test uses the isolation forest algorithm, which builds an ensemble of isolation trees by randomly selecting
30
+ features and splitting the data based on random thresholds. It isolates anomalies rather than focusing on normal
31
+ data points. For each pair of variables, a scatter plot is generated which distinguishes the identified outliers
32
+ from the inliers. The results of the test can be visualized using these scatter plots, illustrating the distinction
33
+ between outliers and inliers.
34
+
35
+ ### Signs of High Risk
36
+
32
37
  - The presence of high contamination, indicating a large number of anomalies
33
38
  - Inability to detect clusters of anomalies that are close in the feature space
34
39
  - Misclassifying normal instances as anomalies
35
40
  - Failure to detect actual anomalies
36
41
 
37
- **Strengths**:
42
+ ### Strengths
43
+
38
44
  - Ability to handle large, high-dimensional datasets
39
45
  - Efficiency in isolating anomalies instead of normal instances
40
46
  - Insensitivity to the underlying distribution of data
@@ -42,7 +48,8 @@ class IsolationForestOutliers(Metric):
42
48
  distinctive properties
43
49
  - Visually presents the test results for better understanding and interpretability
44
50
 
45
- **Limitations**:
51
+ ### Limitations
52
+
46
53
  - Difficult to detect anomalies that are close to each other or prevalent in datasets
47
54
  - Dependency on the contamination parameter which may need fine-tuning to be effective
48
55
  - Potential failure in detecting collective anomalies if they behave similarly to normal data