validmind 2.5.8__py3-none-any.whl → 2.5.18__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- validmind/__version__.py +1 -1
- validmind/ai/test_descriptions.py +80 -119
- validmind/ai/test_result_description/config.yaml +29 -0
- validmind/ai/test_result_description/context.py +73 -0
- validmind/ai/test_result_description/image_processing.py +124 -0
- validmind/ai/test_result_description/system.jinja +39 -0
- validmind/ai/test_result_description/user.jinja +25 -0
- validmind/api_client.py +89 -43
- validmind/client.py +2 -2
- validmind/client_config.py +11 -14
- validmind/datasets/credit_risk/__init__.py +1 -0
- validmind/datasets/credit_risk/datasets/lending_club_biased.csv.gz +0 -0
- validmind/datasets/credit_risk/lending_club_bias.py +142 -0
- validmind/datasets/regression/fred_timeseries.py +67 -138
- validmind/template.py +1 -0
- validmind/test_suites/__init__.py +0 -2
- validmind/test_suites/statsmodels_timeseries.py +1 -1
- validmind/test_suites/summarization.py +0 -1
- validmind/test_suites/time_series.py +0 -43
- validmind/tests/__types__.py +14 -15
- validmind/tests/data_validation/ACFandPACFPlot.py +15 -13
- validmind/tests/data_validation/ADF.py +31 -24
- validmind/tests/data_validation/AutoAR.py +9 -9
- validmind/tests/data_validation/AutoMA.py +23 -16
- validmind/tests/data_validation/AutoSeasonality.py +18 -16
- validmind/tests/data_validation/AutoStationarity.py +21 -16
- validmind/tests/data_validation/BivariateScatterPlots.py +67 -96
- validmind/tests/{model_validation/statsmodels → data_validation}/BoxPierce.py +34 -34
- validmind/tests/data_validation/ChiSquaredFeaturesTable.py +85 -124
- validmind/tests/data_validation/ClassImbalance.py +15 -12
- validmind/tests/data_validation/DFGLSArch.py +19 -13
- validmind/tests/data_validation/DatasetDescription.py +17 -11
- validmind/tests/data_validation/DatasetSplit.py +7 -5
- validmind/tests/data_validation/DescriptiveStatistics.py +28 -21
- validmind/tests/data_validation/Duplicates.py +33 -25
- validmind/tests/data_validation/EngleGrangerCoint.py +35 -33
- validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +59 -71
- validmind/tests/data_validation/HighCardinality.py +19 -12
- validmind/tests/data_validation/HighPearsonCorrelation.py +27 -22
- validmind/tests/data_validation/IQROutliersBarPlot.py +13 -10
- validmind/tests/data_validation/IQROutliersTable.py +40 -36
- validmind/tests/data_validation/IsolationForestOutliers.py +21 -14
- validmind/tests/data_validation/JarqueBera.py +70 -0
- validmind/tests/data_validation/KPSS.py +34 -29
- validmind/tests/data_validation/LJungBox.py +66 -0
- validmind/tests/data_validation/LaggedCorrelationHeatmap.py +22 -15
- validmind/tests/data_validation/MissingValues.py +32 -27
- validmind/tests/data_validation/MissingValuesBarPlot.py +25 -21
- validmind/tests/data_validation/PearsonCorrelationMatrix.py +71 -84
- validmind/tests/data_validation/PhillipsPerronArch.py +37 -30
- validmind/tests/data_validation/ProtectedClassesCombination.py +197 -0
- validmind/tests/data_validation/ProtectedClassesDescription.py +130 -0
- validmind/tests/data_validation/ProtectedClassesDisparity.py +133 -0
- validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py +172 -0
- validmind/tests/data_validation/RollingStatsPlot.py +31 -23
- validmind/tests/data_validation/RunsTest.py +72 -0
- validmind/tests/data_validation/ScatterPlot.py +63 -78
- validmind/tests/data_validation/SeasonalDecompose.py +38 -34
- validmind/tests/{model_validation/statsmodels → data_validation}/ShapiroWilk.py +35 -30
- validmind/tests/data_validation/Skewness.py +35 -37
- validmind/tests/data_validation/SpreadPlot.py +35 -35
- validmind/tests/data_validation/TabularCategoricalBarPlots.py +23 -17
- validmind/tests/data_validation/TabularDateTimeHistograms.py +21 -13
- validmind/tests/data_validation/TabularDescriptionTables.py +51 -16
- validmind/tests/data_validation/TabularNumericalHistograms.py +25 -22
- validmind/tests/data_validation/TargetRateBarPlots.py +21 -14
- validmind/tests/data_validation/TimeSeriesDescription.py +25 -18
- validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py +23 -17
- validmind/tests/data_validation/TimeSeriesFrequency.py +24 -17
- validmind/tests/data_validation/TimeSeriesHistogram.py +33 -32
- validmind/tests/data_validation/TimeSeriesLinePlot.py +17 -10
- validmind/tests/data_validation/TimeSeriesMissingValues.py +15 -10
- validmind/tests/data_validation/TimeSeriesOutliers.py +37 -33
- validmind/tests/data_validation/TooManyZeroValues.py +16 -11
- validmind/tests/data_validation/UniqueRows.py +11 -6
- validmind/tests/data_validation/WOEBinPlots.py +23 -16
- validmind/tests/data_validation/WOEBinTable.py +35 -30
- validmind/tests/data_validation/ZivotAndrewsArch.py +34 -28
- validmind/tests/data_validation/nlp/CommonWords.py +21 -14
- validmind/tests/data_validation/nlp/Hashtags.py +42 -40
- validmind/tests/data_validation/nlp/LanguageDetection.py +33 -14
- validmind/tests/data_validation/nlp/Mentions.py +21 -15
- validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +32 -9
- validmind/tests/data_validation/nlp/Punctuations.py +24 -20
- validmind/tests/data_validation/nlp/Sentiment.py +27 -8
- validmind/tests/data_validation/nlp/StopWords.py +26 -19
- validmind/tests/data_validation/nlp/TextDescription.py +39 -36
- validmind/tests/data_validation/nlp/Toxicity.py +32 -9
- validmind/tests/decorator.py +81 -42
- validmind/tests/model_validation/BertScore.py +36 -27
- validmind/tests/model_validation/BleuScore.py +25 -19
- validmind/tests/model_validation/ClusterSizeDistribution.py +38 -34
- validmind/tests/model_validation/ContextualRecall.py +38 -13
- validmind/tests/model_validation/FeaturesAUC.py +32 -13
- validmind/tests/model_validation/MeteorScore.py +46 -33
- validmind/tests/model_validation/ModelMetadata.py +32 -64
- validmind/tests/model_validation/ModelPredictionResiduals.py +75 -73
- validmind/tests/model_validation/RegardScore.py +30 -14
- validmind/tests/model_validation/RegressionResidualsPlot.py +10 -5
- validmind/tests/model_validation/RougeScore.py +36 -30
- validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +30 -14
- validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +27 -30
- validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +68 -63
- validmind/tests/model_validation/TokenDisparity.py +31 -23
- validmind/tests/model_validation/ToxicityScore.py +26 -17
- validmind/tests/model_validation/embeddings/ClusterDistribution.py +24 -20
- validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +30 -27
- validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +7 -5
- validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +32 -23
- validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +7 -5
- validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +15 -11
- validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +29 -29
- validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +34 -25
- validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +38 -26
- validmind/tests/model_validation/embeddings/StabilityAnalysis.py +40 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +18 -17
- validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +40 -45
- validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +17 -19
- validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +29 -25
- validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +38 -28
- validmind/tests/model_validation/ragas/AnswerCorrectness.py +5 -4
- validmind/tests/model_validation/ragas/AnswerRelevance.py +5 -4
- validmind/tests/model_validation/ragas/AnswerSimilarity.py +5 -4
- validmind/tests/model_validation/ragas/AspectCritique.py +12 -6
- validmind/tests/model_validation/ragas/ContextEntityRecall.py +9 -8
- validmind/tests/model_validation/ragas/ContextPrecision.py +5 -4
- validmind/tests/model_validation/ragas/ContextRecall.py +5 -4
- validmind/tests/model_validation/ragas/ContextUtilization.py +155 -0
- validmind/tests/model_validation/ragas/Faithfulness.py +5 -4
- validmind/tests/model_validation/ragas/NoiseSensitivity.py +152 -0
- validmind/tests/model_validation/ragas/utils.py +6 -0
- validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +19 -12
- validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +22 -17
- validmind/tests/model_validation/sklearn/ClassifierPerformance.py +27 -25
- validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +7 -5
- validmind/tests/model_validation/sklearn/ClusterPerformance.py +40 -78
- validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +15 -17
- validmind/tests/model_validation/sklearn/CompletenessScore.py +17 -11
- validmind/tests/model_validation/sklearn/ConfusionMatrix.py +22 -15
- validmind/tests/model_validation/sklearn/FeatureImportance.py +95 -0
- validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +7 -7
- validmind/tests/model_validation/sklearn/HomogeneityScore.py +19 -12
- validmind/tests/model_validation/sklearn/HyperParametersTuning.py +35 -30
- validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +10 -5
- validmind/tests/model_validation/sklearn/MinimumAccuracy.py +32 -32
- validmind/tests/model_validation/sklearn/MinimumF1Score.py +23 -23
- validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +15 -10
- validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py +26 -19
- validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +38 -18
- validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +32 -26
- validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +8 -6
- validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +24 -17
- validmind/tests/model_validation/sklearn/ROCCurve.py +12 -7
- validmind/tests/model_validation/sklearn/RegressionErrors.py +74 -130
- validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py +27 -12
- validmind/tests/model_validation/sklearn/{RegressionModelsPerformanceComparison.py → RegressionPerformance.py} +18 -20
- validmind/tests/model_validation/sklearn/RegressionR2Square.py +55 -94
- validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py +32 -13
- validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +36 -32
- validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py +66 -5
- validmind/tests/model_validation/sklearn/SilhouettePlot.py +27 -19
- validmind/tests/model_validation/sklearn/TrainingTestDegradation.py +25 -18
- validmind/tests/model_validation/sklearn/VMeasure.py +14 -13
- validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +7 -5
- validmind/tests/model_validation/statsmodels/AutoARIMA.py +24 -18
- validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +73 -104
- validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +59 -32
- validmind/tests/model_validation/statsmodels/GINITable.py +44 -77
- validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +33 -34
- validmind/tests/model_validation/statsmodels/Lilliefors.py +27 -24
- validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +86 -119
- validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +100 -0
- validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +14 -9
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +17 -13
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +46 -43
- validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +38 -36
- validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +30 -28
- validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +18 -11
- validmind/tests/model_validation/statsmodels/ScorecardHistogram.py +75 -107
- validmind/tests/ongoing_monitoring/FeatureDrift.py +10 -6
- validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +31 -25
- validmind/tests/ongoing_monitoring/PredictionCorrelation.py +29 -21
- validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +31 -23
- validmind/tests/prompt_validation/Bias.py +14 -11
- validmind/tests/prompt_validation/Clarity.py +16 -14
- validmind/tests/prompt_validation/Conciseness.py +7 -5
- validmind/tests/prompt_validation/Delimitation.py +23 -22
- validmind/tests/prompt_validation/NegativeInstruction.py +7 -5
- validmind/tests/prompt_validation/Robustness.py +12 -10
- validmind/tests/prompt_validation/Specificity.py +13 -11
- validmind/tests/prompt_validation/ai_powered_test.py +6 -0
- validmind/tests/run.py +68 -23
- validmind/unit_metrics/__init__.py +81 -144
- validmind/unit_metrics/classification/{sklearn/Accuracy.py → Accuracy.py} +1 -1
- validmind/unit_metrics/classification/{sklearn/F1.py → F1.py} +1 -1
- validmind/unit_metrics/classification/{sklearn/Precision.py → Precision.py} +1 -1
- validmind/unit_metrics/classification/{sklearn/ROC_AUC.py → ROC_AUC.py} +1 -2
- validmind/unit_metrics/classification/{sklearn/Recall.py → Recall.py} +1 -1
- validmind/unit_metrics/regression/{sklearn/AdjustedRSquaredScore.py → AdjustedRSquaredScore.py} +1 -1
- validmind/unit_metrics/regression/GiniCoefficient.py +1 -1
- validmind/unit_metrics/regression/HuberLoss.py +1 -1
- validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py +1 -1
- validmind/unit_metrics/regression/{sklearn/MeanAbsoluteError.py → MeanAbsoluteError.py} +1 -1
- validmind/unit_metrics/regression/MeanAbsolutePercentageError.py +1 -1
- validmind/unit_metrics/regression/MeanBiasDeviation.py +1 -1
- validmind/unit_metrics/regression/{sklearn/MeanSquaredError.py → MeanSquaredError.py} +1 -1
- validmind/unit_metrics/regression/QuantileLoss.py +1 -1
- validmind/unit_metrics/regression/{sklearn/RSquaredScore.py → RSquaredScore.py} +1 -1
- validmind/unit_metrics/regression/{sklearn/RootMeanSquaredError.py → RootMeanSquaredError.py} +1 -1
- validmind/utils.py +4 -0
- validmind/vm_models/dataset/dataset.py +2 -0
- validmind/vm_models/figure.py +5 -0
- validmind/vm_models/test/metric.py +1 -0
- validmind/vm_models/test/result_wrapper.py +143 -158
- validmind/vm_models/test/threshold_test.py +1 -0
- {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/METADATA +4 -3
- validmind-2.5.18.dist-info/RECORD +324 -0
- validmind/tests/data_validation/ANOVAOneWayTable.py +0 -138
- validmind/tests/data_validation/BivariateFeaturesBarPlots.py +0 -142
- validmind/tests/data_validation/BivariateHistograms.py +0 -117
- validmind/tests/data_validation/HeatmapFeatureCorrelations.py +0 -124
- validmind/tests/data_validation/MissingValuesRisk.py +0 -88
- validmind/tests/model_validation/ModelMetadataComparison.py +0 -59
- validmind/tests/model_validation/sklearn/FeatureImportanceComparison.py +0 -83
- validmind/tests/model_validation/statsmodels/JarqueBera.py +0 -73
- validmind/tests/model_validation/statsmodels/LJungBox.py +0 -66
- validmind/tests/model_validation/statsmodels/RegressionCoeffsPlot.py +0 -135
- validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py +0 -103
- validmind/tests/model_validation/statsmodels/RunsTest.py +0 -71
- validmind-2.5.8.dist-info/RECORD +0 -318
- {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/LICENSE +0 -0
- {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/WHEEL +0 -0
- {validmind-2.5.8.dist-info → validmind-2.5.18.dist-info}/entry_points.txt +0 -0
@@ -10,41 +10,43 @@ from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableM
|
|
10
10
|
|
11
11
|
class EngleGrangerCoint(Metric):
|
12
12
|
"""
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
over time.
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
13
|
+
Assesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test.
|
14
|
+
|
15
|
+
### Purpose
|
16
|
+
|
17
|
+
The intent of this Engle-Granger cointegration test is to explore and quantify the degree of co-movement between
|
18
|
+
pairs of time series variables in a dataset. This is particularly useful in enhancing the accuracy of predictive
|
19
|
+
regressions whenever the underlying variables are co-integrated, i.e., they move together over time.
|
20
|
+
|
21
|
+
### Test Mechanism
|
22
|
+
|
23
|
+
The test first drops any non-applicable values from the input dataset and then iterates over each pair of variables
|
24
|
+
to apply the Engle-Granger cointegration test. The test generates a 'p' value, which is then compared against a
|
25
|
+
pre-specified threshold (0.05 by default). The pair is labeled as 'Cointegrated' if the 'p' value is less than or
|
26
|
+
equal to the threshold or 'Not cointegrated' otherwise. A summary table is returned by the metric showing
|
27
|
+
cointegration results for each variable pair.
|
28
|
+
|
29
|
+
### Signs of High Risk
|
30
|
+
|
31
|
+
- A significant number of hypothesized cointegrated variables do not pass the test.
|
32
|
+
- A considerable number of 'p' values are close to the threshold, indicating minor data fluctuations can switch the
|
33
|
+
decision between 'Cointegrated' and 'Not cointegrated'.
|
34
|
+
|
35
|
+
### Strengths
|
36
|
+
|
37
|
+
- Provides an effective way to analyze relationships between time series, particularly in contexts where it's
|
38
|
+
essential to check if variables move together in a statistically significant manner.
|
39
|
+
- Useful in various domains, especially finance or economics, where predictive models often hinge on understanding
|
40
|
+
how different variables move together over time.
|
41
|
+
|
42
|
+
### Limitations
|
43
|
+
|
44
|
+
- Assumes that the time series are integrated of the same order, which isn't always true in multivariate time
|
45
|
+
series datasets.
|
43
46
|
- The presence of non-stationary characteristics in the series or structural breaks can result in falsely positive
|
44
47
|
or negative cointegration results.
|
45
|
-
-
|
46
|
-
|
47
|
-
evaluation.
|
48
|
+
- May not perform well for small sample sizes due to lack of statistical power and should be supplemented with
|
49
|
+
other predictive indicators for a more robust model evaluation.
|
48
50
|
"""
|
49
51
|
|
50
52
|
type = "dataset"
|
@@ -2,108 +2,96 @@
|
|
2
2
|
# See the LICENSE file in the root of this repository for details.
|
3
3
|
# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
|
4
4
|
|
5
|
-
from dataclasses import dataclass
|
6
5
|
|
7
6
|
import numpy as np
|
8
7
|
import plotly.graph_objects as go
|
9
8
|
|
10
|
-
from validmind
|
9
|
+
from validmind import tags, tasks
|
11
10
|
|
12
11
|
|
13
|
-
@
|
14
|
-
|
12
|
+
@tags("tabular_data", "visualization", "correlation")
|
13
|
+
@tasks("classification", "regression")
|
14
|
+
def FeatureTargetCorrelationPlot(dataset, fig_height=600):
|
15
15
|
"""
|
16
|
-
Visualizes the correlation between input features and model's target output in a color-coded horizontal bar
|
16
|
+
Visualizes the correlation between input features and the model's target output in a color-coded horizontal bar
|
17
|
+
plot.
|
17
18
|
|
18
|
-
|
19
|
-
the target output of a Machine Learning model. Understanding how each feature influences the model's predictions is
|
20
|
-
crucial - a higher correlation indicates stronger influence of the feature on the target variable. This correlation
|
21
|
-
study is especially advantageous during feature selection and for comprehending the model's operation.
|
19
|
+
### Purpose
|
22
20
|
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
21
|
+
This test is designed to graphically illustrate the correlations between distinct input features and the target
|
22
|
+
output of a Machine Learning model. Understanding how each feature influences the model's predictions is crucial—a
|
23
|
+
higher correlation indicates a stronger influence of the feature on the target variable. This correlation study is
|
24
|
+
especially advantageous during feature selection and for comprehending the model's operation.
|
25
|
+
|
26
|
+
### Test Mechanism
|
27
|
+
|
28
|
+
This FeatureTargetCorrelationPlot test computes and presents the correlations between the features and the target
|
29
|
+
variable using a specific dataset. These correlations are calculated and are then graphically represented in a
|
30
|
+
horizontal bar plot, color-coded based on the strength of the correlation. A hovering template can also be utilized
|
31
|
+
for informative tooltips. It is possible to specify the features to be analyzed and adjust the graph's height
|
32
|
+
according to need.
|
33
|
+
|
34
|
+
### Signs of High Risk
|
28
35
|
|
29
|
-
**Signs of High Risk**:
|
30
36
|
- There are no strong correlations (either positive or negative) between features and the target variable. This
|
31
37
|
could suggest high risk as the supplied features do not appear to significantly impact the prediction output.
|
32
38
|
- The presence of duplicated correlation values might hint at redundancy in the feature set.
|
33
39
|
|
34
|
-
|
40
|
+
### Strengths
|
41
|
+
|
35
42
|
- Provides visual assistance to interpreting correlations more effectively.
|
36
43
|
- Gives a clear and simple tour of how each feature affects the model's target variable.
|
37
44
|
- Beneficial for feature selection and grasping the model's prediction nature.
|
38
45
|
- Precise correlation values for each feature are offered by the hover template, contributing to a granular-level
|
39
46
|
comprehension.
|
40
47
|
|
41
|
-
|
48
|
+
### Limitations
|
49
|
+
|
42
50
|
- The test only accepts numerical data, meaning variables of other types need to be prepared beforehand.
|
43
51
|
- The plot assumes all correlations to be linear, thus non-linear relationships might not be captured effectively.
|
44
52
|
- Not apt for models that employ complex feature interactions, like Decision Trees or Neural Networks, as the test
|
45
53
|
may not accurately reflect their importance.
|
46
54
|
"""
|
47
55
|
|
48
|
-
|
49
|
-
|
50
|
-
default_params = {"features": None, "fig_height": 600}
|
51
|
-
tasks = ["classification", "regression"]
|
52
|
-
tags = ["tabular_data", "visualization", "feature_importance", "correlation"]
|
56
|
+
# Filter DataFrame based on features and target_column
|
57
|
+
df = dataset.df[dataset.feature_columns + [dataset.target_column]]
|
53
58
|
|
54
|
-
|
55
|
-
fig_height = self.params["fig_height"]
|
59
|
+
fig = _visualize_feature_target_correlation(df, dataset.target_column, fig_height)
|
56
60
|
|
57
|
-
|
58
|
-
features = self.inputs.dataset.feature_columns
|
59
|
-
else:
|
60
|
-
features = self.params["features"]
|
61
|
+
return fig
|
61
62
|
|
62
|
-
target_column = self.inputs.dataset.target_column
|
63
63
|
|
64
|
-
|
65
|
-
|
64
|
+
def _visualize_feature_target_correlation(df, target_column, fig_height):
|
65
|
+
# Compute correlations with the target variable
|
66
|
+
correlations = (
|
67
|
+
df.corr(numeric_only=True)[target_column].drop(target_column).to_frame()
|
68
|
+
)
|
69
|
+
correlations = correlations.loc[:, ~correlations.columns.duplicated()]
|
66
70
|
|
67
|
-
|
68
|
-
df, target_column, fig_height
|
69
|
-
)
|
71
|
+
correlations = correlations.sort_values(by=target_column, ascending=True)
|
70
72
|
|
71
|
-
|
73
|
+
# Create a gradual color map from red (1) to blue (-1)
|
74
|
+
color_map = np.linspace(1, -1, len(correlations))
|
75
|
+
colors = [
|
76
|
+
f"rgb({int(255 * (1 - val))}, 0, {int(255 * (1 + val))})" for val in color_map
|
77
|
+
]
|
72
78
|
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
# Create a gradual color map from red (1) to blue (-1)
|
83
|
-
color_map = np.linspace(1, -1, len(correlations))
|
84
|
-
colors = [
|
85
|
-
f"rgb({int(255 * (1 - val))}, 0, {int(255 * (1 + val))})"
|
86
|
-
for val in color_map
|
87
|
-
]
|
88
|
-
|
89
|
-
# Create a horizontal bar plot with gradual color mapping
|
90
|
-
fig = go.Figure(
|
91
|
-
data=go.Bar(
|
92
|
-
x=correlations[target_column],
|
93
|
-
y=correlations.index,
|
94
|
-
orientation="h",
|
95
|
-
marker=dict(color=colors),
|
96
|
-
hovertemplate="Feature: %{y}<br>Correlation: %{x:.2f}<extra></extra>", # Hover template for tooltips
|
97
|
-
)
|
79
|
+
# Create a horizontal bar plot with gradual color mapping
|
80
|
+
fig = go.Figure(
|
81
|
+
data=go.Bar(
|
82
|
+
x=correlations[target_column],
|
83
|
+
y=correlations.index,
|
84
|
+
orientation="h",
|
85
|
+
marker=dict(color=colors),
|
86
|
+
hovertemplate="Feature: %{y}<br>Correlation: %{x:.2f}<extra></extra>", # Hover template for tooltips
|
98
87
|
)
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
return [figure]
|
88
|
+
)
|
89
|
+
|
90
|
+
# Set the title and axis labels
|
91
|
+
fig.update_layout(
|
92
|
+
title=f"Correlations of Features vs Target Variable ({target_column})",
|
93
|
+
xaxis_title="",
|
94
|
+
yaxis_title="",
|
95
|
+
height=fig_height, # Adjust the height value as needed
|
96
|
+
)
|
97
|
+
return fig
|
@@ -22,26 +22,33 @@ class HighCardinality(ThresholdTest):
|
|
22
22
|
"""
|
23
23
|
Assesses the number of unique values in categorical columns to detect high cardinality and potential overfitting.
|
24
24
|
|
25
|
-
|
26
|
-
columns of a dataset. In this context, high cardinality implies the presence of a large number of unique,
|
27
|
-
non-repetitive values in the dataset.
|
25
|
+
### Purpose
|
28
26
|
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
27
|
+
The “High Cardinality” test is used to evaluate the number of unique values present in the categorical columns of a
|
28
|
+
dataset. In this context, high cardinality implies the presence of a large number of unique, non-repetitive values
|
29
|
+
in the dataset.
|
30
|
+
|
31
|
+
### Test Mechanism
|
32
|
+
|
33
|
+
The test first infers the dataset's type and then calculates an initial numeric threshold based on the test
|
34
|
+
parameters. It only considers columns classified as "Categorical". For each of these columns, the number of
|
35
|
+
distinct values (n_distinct) and the percentage of distinct values (p_distinct) are calculated. The test will pass
|
36
|
+
if n_distinct is less than the calculated numeric threshold. Lastly, the results, which include details such as
|
37
|
+
column name, number of distinct values, and pass/fail status, are compiled into a table.
|
38
|
+
|
39
|
+
### Signs of High Risk
|
34
40
|
|
35
|
-
**Signs of High Risk**:
|
36
41
|
- A large number of distinct values (high cardinality) in one or more categorical columns implies a high risk.
|
37
42
|
- A column failing the test (n_distinct >= num_threshold) is another indicator of high risk.
|
38
43
|
|
39
|
-
|
44
|
+
### Strengths
|
45
|
+
|
40
46
|
- The High Cardinality test is effective in early detection of potential overfitting and unwanted noise.
|
41
47
|
- It aids in identifying potential outliers and inconsistencies, thereby improving data quality.
|
42
|
-
- The test can be applied to both
|
48
|
+
- The test can be applied to both classification and regression task types, demonstrating its versatility.
|
49
|
+
|
50
|
+
### Limitations
|
43
51
|
|
44
|
-
**Limitations**:
|
45
52
|
- The test is restricted to only "Categorical" data types and is thus not suitable for numerical or continuous
|
46
53
|
features, limiting its scope.
|
47
54
|
- The test does not consider the relevance or importance of unique values in categorical features, potentially
|
@@ -22,36 +22,41 @@ class HighPearsonCorrelation(ThresholdTest):
|
|
22
22
|
"""
|
23
23
|
Identifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity.
|
24
24
|
|
25
|
-
|
26
|
-
the main goal of identifying high correlations that might indicate feature redundancy or multicollinearity.
|
27
|
-
Identification of such issue allows developers and risk management teams to properly deal with potential impacts on
|
28
|
-
the machine learning model's performance and interpretability.
|
25
|
+
### Purpose
|
29
26
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
27
|
+
The High Pearson Correlation test measures the linear relationship between features in a dataset, with the main
|
28
|
+
goal of identifying high correlations that might indicate feature redundancy or multicollinearity. Identification
|
29
|
+
of such issues allows developers and risk management teams to properly deal with potential impacts on the machine
|
30
|
+
learning model's performance and interpretability.
|
31
|
+
|
32
|
+
### Test Mechanism
|
33
|
+
|
34
|
+
The test works by generating pairwise Pearson correlations for all features in the dataset, then sorting and
|
35
|
+
eliminating duplicate and self-correlations. It assigns a Pass or Fail based on whether the absolute value of the
|
36
|
+
correlation coefficient surpasses a pre-set threshold (defaulted at 0.3). It lastly returns the top ten strongest
|
37
|
+
correlations regardless of passing or failing status.
|
38
|
+
|
39
|
+
### Signs of High Risk
|
34
40
|
|
35
|
-
**Signs of High Risk**:
|
36
41
|
- A high risk indication would be the presence of correlation coefficients exceeding the threshold.
|
37
42
|
- If the features share a strong linear relationship, this could lead to potential multicollinearity and model
|
38
43
|
overfitting.
|
39
44
|
- Redundancy of variables can undermine the interpretability of the model due to uncertainty over the authenticity
|
40
45
|
of individual variable's predictive power.
|
41
46
|
|
42
|
-
|
43
|
-
|
44
|
-
pairs.
|
45
|
-
-
|
46
|
-
|
47
|
-
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
relationships or dependencies.
|
52
|
-
-
|
53
|
-
-
|
54
|
-
|
47
|
+
### Strengths
|
48
|
+
|
49
|
+
- Provides a quick and simple means of identifying relationships between feature pairs.
|
50
|
+
- Generates a transparent output that displays pairs of correlated variables, the Pearson correlation coefficient,
|
51
|
+
and a Pass or Fail status for each.
|
52
|
+
- Aids in early identification of potential multicollinearity issues that may disrupt model training.
|
53
|
+
|
54
|
+
### Limitations
|
55
|
+
|
56
|
+
- Can only delineate linear relationships, failing to shed light on nonlinear relationships or dependencies.
|
57
|
+
- Sensitive to outliers where a few outliers could notably affect the correlation coefficient.
|
58
|
+
- Limited to identifying redundancy only within feature pairs; may fail to spot more complex relationships among
|
59
|
+
three or more variables.
|
55
60
|
- The top 10 result filter might not fully capture the richness of the data; an option to configure the number of
|
56
61
|
retained results could be helpful.
|
57
62
|
"""
|
@@ -12,14 +12,18 @@ from validmind.vm_models import Figure, Metric
|
|
12
12
|
@dataclass
|
13
13
|
class IQROutliersBarPlot(Metric):
|
14
14
|
"""
|
15
|
-
Visualizes outlier distribution across percentiles in numerical data using Interquartile Range (IQR) method.
|
15
|
+
Visualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method.
|
16
16
|
|
17
|
-
|
18
|
-
evaluate the extent of outliers in numeric variables based on percentiles. Its primary purpose is to clarify the
|
19
|
-
dataset's distribution, flag possible abnormalities in it and gauge potential risks associated with processing
|
20
|
-
potentially skewed data, which can affect the machine learning model's predictive prowess.
|
17
|
+
### Purpose
|
21
18
|
|
22
|
-
|
19
|
+
The InterQuartile Range Outliers Bar Plot (IQROutliersBarPlot) metric aims to visually analyze and evaluate the
|
20
|
+
extent of outliers in numeric variables based on percentiles. Its primary purpose is to clarify the dataset's
|
21
|
+
distribution, flag possible abnormalities in it, and gauge potential risks associated with processing potentially
|
22
|
+
skewed data, which can affect the machine learning model's predictive prowess.
|
23
|
+
|
24
|
+
### Test Mechanism
|
25
|
+
|
26
|
+
The examination invokes a series of steps:
|
23
27
|
|
24
28
|
1. For every numeric feature in the dataset, the 25th percentile (Q1) and 75th percentile (Q3) are calculated
|
25
29
|
before deriving the Interquartile Range (IQR), the difference between Q1 and Q3.
|
@@ -31,8 +35,7 @@ class IQROutliersBarPlot(Metric):
|
|
31
35
|
5. These counts are employed to construct a bar plot for the feature, showcasing the distribution of outliers
|
32
36
|
across different percentiles.
|
33
37
|
|
34
|
-
|
35
|
-
following signs:
|
38
|
+
### Signs of High Risk
|
36
39
|
|
37
40
|
- A prevalence of outliers in the data, potentially skewing its distribution.
|
38
41
|
- Outliers dominating higher percentiles (75-100) which implies the presence of extreme values, capable of severely
|
@@ -40,7 +43,7 @@ class IQROutliersBarPlot(Metric):
|
|
40
43
|
- Certain features harboring most of their values as outliers, which signifies that these features might not
|
41
44
|
contribute positively to the model's forecasting ability.
|
42
45
|
|
43
|
-
|
46
|
+
### Strengths
|
44
47
|
|
45
48
|
- Effectively identifies outliers in the data through visual means, facilitating easier comprehension and offering
|
46
49
|
insights into the outliers' possible impact on the model.
|
@@ -48,7 +51,7 @@ class IQROutliersBarPlot(Metric):
|
|
48
51
|
- Task-agnostic in nature; it is viable for both classification and regression tasks.
|
49
52
|
- Can handle large datasets as its operation does not hinge on computationally heavy operations.
|
50
53
|
|
51
|
-
|
54
|
+
### Limitations
|
52
55
|
|
53
56
|
- Its application is limited to numerical variables and does not extend to categorical ones.
|
54
57
|
- Relies on a predefined threshold (default being 1.5) for outlier identification, which may not be suitable for
|
@@ -12,42 +12,46 @@ from validmind.vm_models import Metric, ResultSummary, ResultTable, ResultTableM
|
|
12
12
|
@dataclass
|
13
13
|
class IQROutliersTable(Metric):
|
14
14
|
"""
|
15
|
-
Determines and summarizes outliers in numerical features using Interquartile Range method.
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
-
|
49
|
-
|
50
|
-
|
15
|
+
Determines and summarizes outliers in numerical features using the Interquartile Range method.
|
16
|
+
|
17
|
+
### Purpose
|
18
|
+
|
19
|
+
The "Interquartile Range Outliers Table" (IQROutliersTable) metric is designed to identify and summarize outliers
|
20
|
+
within numerical features of a dataset using the Interquartile Range (IQR) method. This exercise is crucial in the
|
21
|
+
pre-processing of data because outliers can substantially distort statistical analysis and impact the performance
|
22
|
+
of machine learning models.
|
23
|
+
|
24
|
+
### Test Mechanism
|
25
|
+
|
26
|
+
The IQR, which is the range separating the first quartile (25th percentile) from the third quartile (75th
|
27
|
+
percentile), is calculated for each numerical feature within the dataset. An outlier is defined as a data point
|
28
|
+
falling below the "Q1 - 1.5 * IQR" or above "Q3 + 1.5 * IQR" range. The test computes the number of outliers and
|
29
|
+
their summary statistics (minimum, 25th percentile, median, 75th percentile, and maximum values) for each numerical
|
30
|
+
feature. If no specific features are chosen, the test applies to all numerical features in the dataset. The default
|
31
|
+
outlier threshold is set to 1.5 but can be customized by the user.
|
32
|
+
|
33
|
+
### Signs of High Risk
|
34
|
+
|
35
|
+
- A large number of outliers in multiple features.
|
36
|
+
- Outliers significantly distanced from the mean value of variables.
|
37
|
+
- Extremely high or low outlier values indicative of data entry errors or other data quality issues.
|
38
|
+
|
39
|
+
### Strengths
|
40
|
+
|
41
|
+
- Provides a comprehensive summary of outliers for each numerical feature, helping pinpoint features with potential
|
42
|
+
quality issues.
|
43
|
+
- The IQR method is robust to extremely high or low outlier values as it is based on quartile calculations.
|
44
|
+
- Can be customized to work on selected features and set thresholds for outliers.
|
45
|
+
|
46
|
+
### Limitations
|
47
|
+
|
48
|
+
- Might cause false positives if the variable deviates from a normal or near-normal distribution, especially for
|
49
|
+
skewed distributions.
|
50
|
+
- Does not provide interpretation or recommendations for addressing outliers, relying on further analysis by users
|
51
|
+
or data scientists.
|
52
|
+
- Only applicable to numerical features, not categorical data.
|
53
|
+
- Default thresholds may not be optimal for data with heavy pre-processing, manipulation, or inherently high
|
54
|
+
kurtosis (heavy tails).
|
51
55
|
"""
|
52
56
|
|
53
57
|
name = "iqr_outliers_table"
|
@@ -17,24 +17,30 @@ class IsolationForestOutliers(Metric):
|
|
17
17
|
"""
|
18
18
|
Detects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots.
|
19
19
|
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
20
|
+
### Purpose
|
21
|
+
|
22
|
+
The IsolationForestOutliers test is designed to identify anomalies or outliers in the model's dataset using the
|
23
|
+
isolation forest algorithm. This algorithm assumes that anomalous data points can be isolated more quickly due to
|
24
|
+
their distinctive properties. By creating isolation trees and identifying instances with shorter average path
|
25
|
+
lengths, the test is able to pick out data points that differ from the majority.
|
26
|
+
|
27
|
+
### Test Mechanism
|
28
|
+
|
29
|
+
The test uses the isolation forest algorithm, which builds an ensemble of isolation trees by randomly selecting
|
30
|
+
features and splitting the data based on random thresholds. It isolates anomalies rather than focusing on normal
|
31
|
+
data points. For each pair of variables, a scatter plot is generated which distinguishes the identified outliers
|
32
|
+
from the inliers. The results of the test can be visualized using these scatter plots, illustrating the distinction
|
33
|
+
between outliers and inliers.
|
34
|
+
|
35
|
+
### Signs of High Risk
|
36
|
+
|
32
37
|
- The presence of high contamination, indicating a large number of anomalies
|
33
38
|
- Inability to detect clusters of anomalies that are close in the feature space
|
34
39
|
- Misclassifying normal instances as anomalies
|
35
40
|
- Failure to detect actual anomalies
|
36
41
|
|
37
|
-
|
42
|
+
### Strengths
|
43
|
+
|
38
44
|
- Ability to handle large, high-dimensional datasets
|
39
45
|
- Efficiency in isolating anomalies instead of normal instances
|
40
46
|
- Insensitivity to the underlying distribution of data
|
@@ -42,7 +48,8 @@ class IsolationForestOutliers(Metric):
|
|
42
48
|
distinctive properties
|
43
49
|
- Visually presents the test results for better understanding and interpretability
|
44
50
|
|
45
|
-
|
51
|
+
### Limitations
|
52
|
+
|
46
53
|
- Difficult to detect anomalies that are close to each other or prevalent in datasets
|
47
54
|
- Dependency on the contamination parameter which may need fine-tuning to be effective
|
48
55
|
- Potential failure in detecting collective anomalies if they behave similarly to normal data
|