validmind 2.5.6__py3-none-any.whl → 2.5.15__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- validmind/__version__.py +1 -1
- validmind/ai/test_descriptions.py +26 -7
- validmind/api_client.py +89 -43
- validmind/client.py +2 -2
- validmind/client_config.py +11 -14
- validmind/datasets/regression/fred_timeseries.py +67 -138
- validmind/template.py +1 -0
- validmind/test_suites/__init__.py +0 -2
- validmind/test_suites/statsmodels_timeseries.py +1 -1
- validmind/test_suites/summarization.py +0 -1
- validmind/test_suites/time_series.py +0 -43
- validmind/tests/__types__.py +3 -13
- validmind/tests/data_validation/ACFandPACFPlot.py +15 -13
- validmind/tests/data_validation/ADF.py +31 -24
- validmind/tests/data_validation/AutoAR.py +9 -9
- validmind/tests/data_validation/AutoMA.py +23 -16
- validmind/tests/data_validation/AutoSeasonality.py +18 -16
- validmind/tests/data_validation/AutoStationarity.py +21 -16
- validmind/tests/data_validation/BivariateScatterPlots.py +67 -96
- validmind/tests/data_validation/ChiSquaredFeaturesTable.py +82 -124
- validmind/tests/data_validation/ClassImbalance.py +15 -12
- validmind/tests/data_validation/DFGLSArch.py +19 -13
- validmind/tests/data_validation/DatasetDescription.py +17 -11
- validmind/tests/data_validation/DatasetSplit.py +7 -5
- validmind/tests/data_validation/DescriptiveStatistics.py +28 -21
- validmind/tests/data_validation/Duplicates.py +33 -25
- validmind/tests/data_validation/EngleGrangerCoint.py +35 -33
- validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +59 -71
- validmind/tests/data_validation/HighCardinality.py +19 -12
- validmind/tests/data_validation/HighPearsonCorrelation.py +27 -22
- validmind/tests/data_validation/IQROutliersBarPlot.py +13 -10
- validmind/tests/data_validation/IQROutliersTable.py +40 -36
- validmind/tests/data_validation/IsolationForestOutliers.py +21 -14
- validmind/tests/data_validation/KPSS.py +34 -29
- validmind/tests/data_validation/LaggedCorrelationHeatmap.py +22 -15
- validmind/tests/data_validation/MissingValues.py +32 -27
- validmind/tests/data_validation/MissingValuesBarPlot.py +25 -21
- validmind/tests/data_validation/PearsonCorrelationMatrix.py +71 -84
- validmind/tests/data_validation/PhillipsPerronArch.py +37 -30
- validmind/tests/data_validation/RollingStatsPlot.py +31 -23
- validmind/tests/data_validation/ScatterPlot.py +63 -78
- validmind/tests/data_validation/SeasonalDecompose.py +38 -34
- validmind/tests/data_validation/Skewness.py +35 -37
- validmind/tests/data_validation/SpreadPlot.py +35 -35
- validmind/tests/data_validation/TabularCategoricalBarPlots.py +23 -17
- validmind/tests/data_validation/TabularDateTimeHistograms.py +21 -13
- validmind/tests/data_validation/TabularDescriptionTables.py +51 -16
- validmind/tests/data_validation/TabularNumericalHistograms.py +25 -22
- validmind/tests/data_validation/TargetRateBarPlots.py +21 -14
- validmind/tests/data_validation/TimeSeriesDescription.py +25 -18
- validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py +23 -17
- validmind/tests/data_validation/TimeSeriesFrequency.py +24 -17
- validmind/tests/data_validation/TimeSeriesHistogram.py +33 -32
- validmind/tests/data_validation/TimeSeriesLinePlot.py +17 -10
- validmind/tests/data_validation/TimeSeriesMissingValues.py +15 -10
- validmind/tests/data_validation/TimeSeriesOutliers.py +37 -33
- validmind/tests/data_validation/TooManyZeroValues.py +16 -11
- validmind/tests/data_validation/UniqueRows.py +11 -6
- validmind/tests/data_validation/WOEBinPlots.py +23 -16
- validmind/tests/data_validation/WOEBinTable.py +35 -30
- validmind/tests/data_validation/ZivotAndrewsArch.py +34 -28
- validmind/tests/data_validation/nlp/CommonWords.py +21 -14
- validmind/tests/data_validation/nlp/Hashtags.py +27 -20
- validmind/tests/data_validation/nlp/LanguageDetection.py +33 -14
- validmind/tests/data_validation/nlp/Mentions.py +21 -15
- validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +32 -9
- validmind/tests/data_validation/nlp/Punctuations.py +24 -20
- validmind/tests/data_validation/nlp/Sentiment.py +27 -8
- validmind/tests/data_validation/nlp/StopWords.py +26 -19
- validmind/tests/data_validation/nlp/TextDescription.py +36 -35
- validmind/tests/data_validation/nlp/Toxicity.py +32 -9
- validmind/tests/decorator.py +81 -42
- validmind/tests/model_validation/BertScore.py +36 -27
- validmind/tests/model_validation/BleuScore.py +25 -19
- validmind/tests/model_validation/ClusterSizeDistribution.py +38 -34
- validmind/tests/model_validation/ContextualRecall.py +35 -13
- validmind/tests/model_validation/FeaturesAUC.py +32 -13
- validmind/tests/model_validation/MeteorScore.py +46 -33
- validmind/tests/model_validation/ModelMetadata.py +32 -64
- validmind/tests/model_validation/ModelPredictionResiduals.py +75 -73
- validmind/tests/model_validation/RegardScore.py +30 -14
- validmind/tests/model_validation/RegressionResidualsPlot.py +10 -5
- validmind/tests/model_validation/RougeScore.py +36 -30
- validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +30 -14
- validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +27 -30
- validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +68 -63
- validmind/tests/model_validation/TokenDisparity.py +31 -23
- validmind/tests/model_validation/ToxicityScore.py +26 -17
- validmind/tests/model_validation/embeddings/ClusterDistribution.py +24 -20
- validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +30 -27
- validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +7 -5
- validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +32 -23
- validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +7 -5
- validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +15 -11
- validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +29 -29
- validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +34 -25
- validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +38 -26
- validmind/tests/model_validation/embeddings/StabilityAnalysis.py +40 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +18 -17
- validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +40 -45
- validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +17 -19
- validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +29 -25
- validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +38 -28
- validmind/tests/model_validation/ragas/AnswerCorrectness.py +5 -4
- validmind/tests/model_validation/ragas/AnswerRelevance.py +5 -4
- validmind/tests/model_validation/ragas/AnswerSimilarity.py +5 -4
- validmind/tests/model_validation/ragas/AspectCritique.py +7 -0
- validmind/tests/model_validation/ragas/ContextEntityRecall.py +9 -8
- validmind/tests/model_validation/ragas/ContextPrecision.py +5 -4
- validmind/tests/model_validation/ragas/ContextRecall.py +5 -4
- validmind/tests/model_validation/ragas/Faithfulness.py +5 -4
- validmind/tests/model_validation/ragas/utils.py +6 -0
- validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +19 -12
- validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +22 -17
- validmind/tests/model_validation/sklearn/ClassifierPerformance.py +27 -25
- validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +7 -5
- validmind/tests/model_validation/sklearn/ClusterPerformance.py +40 -78
- validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +15 -17
- validmind/tests/model_validation/sklearn/CompletenessScore.py +17 -11
- validmind/tests/model_validation/sklearn/ConfusionMatrix.py +22 -15
- validmind/tests/model_validation/sklearn/FeatureImportance.py +95 -0
- validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +7 -7
- validmind/tests/model_validation/sklearn/HomogeneityScore.py +19 -12
- validmind/tests/model_validation/sklearn/HyperParametersTuning.py +35 -30
- validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +10 -5
- validmind/tests/model_validation/sklearn/MinimumAccuracy.py +32 -32
- validmind/tests/model_validation/sklearn/MinimumF1Score.py +23 -23
- validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +15 -10
- validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py +26 -19
- validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +38 -18
- validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +31 -25
- validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +8 -6
- validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +24 -17
- validmind/tests/model_validation/sklearn/ROCCurve.py +12 -7
- validmind/tests/model_validation/sklearn/RegressionErrors.py +74 -130
- validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py +27 -12
- validmind/tests/model_validation/sklearn/{RegressionModelsPerformanceComparison.py → RegressionPerformance.py} +18 -20
- validmind/tests/model_validation/sklearn/RegressionR2Square.py +55 -93
- validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py +32 -13
- validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +113 -73
- validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py +7 -5
- validmind/tests/model_validation/sklearn/SilhouettePlot.py +27 -19
- validmind/tests/model_validation/sklearn/TrainingTestDegradation.py +25 -18
- validmind/tests/model_validation/sklearn/VMeasure.py +14 -13
- validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +7 -5
- validmind/tests/model_validation/statsmodels/AutoARIMA.py +24 -18
- validmind/tests/model_validation/statsmodels/BoxPierce.py +14 -10
- validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +73 -104
- validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +19 -12
- validmind/tests/model_validation/statsmodels/GINITable.py +44 -77
- validmind/tests/model_validation/statsmodels/JarqueBera.py +27 -22
- validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +33 -34
- validmind/tests/model_validation/statsmodels/LJungBox.py +32 -28
- validmind/tests/model_validation/statsmodels/Lilliefors.py +27 -24
- validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +87 -119
- validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +100 -0
- validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +14 -9
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +17 -13
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +46 -43
- validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +38 -36
- validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +30 -28
- validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +18 -11
- validmind/tests/model_validation/statsmodels/RunsTest.py +32 -28
- validmind/tests/model_validation/statsmodels/ScorecardHistogram.py +75 -107
- validmind/tests/model_validation/statsmodels/ShapiroWilk.py +15 -8
- validmind/tests/ongoing_monitoring/FeatureDrift.py +10 -6
- validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +31 -25
- validmind/tests/ongoing_monitoring/PredictionCorrelation.py +29 -21
- validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +31 -23
- validmind/tests/prompt_validation/Bias.py +14 -11
- validmind/tests/prompt_validation/Clarity.py +16 -14
- validmind/tests/prompt_validation/Conciseness.py +7 -5
- validmind/tests/prompt_validation/Delimitation.py +23 -22
- validmind/tests/prompt_validation/NegativeInstruction.py +7 -5
- validmind/tests/prompt_validation/Robustness.py +12 -10
- validmind/tests/prompt_validation/Specificity.py +13 -11
- validmind/tests/prompt_validation/ai_powered_test.py +6 -0
- validmind/tests/run.py +68 -23
- validmind/unit_metrics/__init__.py +81 -144
- validmind/unit_metrics/classification/{sklearn/Accuracy.py → Accuracy.py} +1 -1
- validmind/unit_metrics/classification/{sklearn/F1.py → F1.py} +1 -1
- validmind/unit_metrics/classification/{sklearn/Precision.py → Precision.py} +1 -1
- validmind/unit_metrics/classification/{sklearn/ROC_AUC.py → ROC_AUC.py} +1 -2
- validmind/unit_metrics/classification/{sklearn/Recall.py → Recall.py} +1 -1
- validmind/unit_metrics/regression/{sklearn/AdjustedRSquaredScore.py → AdjustedRSquaredScore.py} +1 -1
- validmind/unit_metrics/regression/GiniCoefficient.py +1 -1
- validmind/unit_metrics/regression/HuberLoss.py +1 -1
- validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py +1 -1
- validmind/unit_metrics/regression/{sklearn/MeanAbsoluteError.py → MeanAbsoluteError.py} +1 -1
- validmind/unit_metrics/regression/MeanAbsolutePercentageError.py +1 -1
- validmind/unit_metrics/regression/MeanBiasDeviation.py +1 -1
- validmind/unit_metrics/regression/{sklearn/MeanSquaredError.py → MeanSquaredError.py} +1 -1
- validmind/unit_metrics/regression/QuantileLoss.py +1 -1
- validmind/unit_metrics/regression/{sklearn/RSquaredScore.py → RSquaredScore.py} +1 -1
- validmind/unit_metrics/regression/{sklearn/RootMeanSquaredError.py → RootMeanSquaredError.py} +1 -1
- validmind/vm_models/dataset/dataset.py +2 -0
- validmind/vm_models/figure.py +5 -0
- validmind/vm_models/test/result_wrapper.py +93 -132
- {validmind-2.5.6.dist-info → validmind-2.5.15.dist-info}/METADATA +1 -1
- {validmind-2.5.6.dist-info → validmind-2.5.15.dist-info}/RECORD +203 -210
- validmind/tests/data_validation/ANOVAOneWayTable.py +0 -138
- validmind/tests/data_validation/BivariateFeaturesBarPlots.py +0 -142
- validmind/tests/data_validation/BivariateHistograms.py +0 -117
- validmind/tests/data_validation/HeatmapFeatureCorrelations.py +0 -124
- validmind/tests/data_validation/MissingValuesRisk.py +0 -88
- validmind/tests/model_validation/ModelMetadataComparison.py +0 -59
- validmind/tests/model_validation/sklearn/FeatureImportanceComparison.py +0 -83
- validmind/tests/model_validation/statsmodels/RegressionCoeffsPlot.py +0 -135
- validmind/tests/model_validation/statsmodels/RegressionModelsCoeffs.py +0 -103
- {validmind-2.5.6.dist-info → validmind-2.5.15.dist-info}/LICENSE +0 -0
- {validmind-2.5.6.dist-info → validmind-2.5.15.dist-info}/WHEEL +0 -0
- {validmind-2.5.6.dist-info → validmind-2.5.15.dist-info}/entry_points.txt +0 -0
@@ -17,36 +17,43 @@ logger = get_logger(__name__)
|
|
17
17
|
@dataclass
|
18
18
|
class PhillipsPerronArch(Metric):
|
19
19
|
"""
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
-
|
34
|
-
|
35
|
-
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
-
|
41
|
-
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
-
|
47
|
-
|
48
|
-
-
|
49
|
-
|
20
|
+
Assesses the stationarity of time series data in each feature of the ML model using the Phillips-Perron test.
|
21
|
+
|
22
|
+
### Purpose
|
23
|
+
|
24
|
+
The Phillips-Perron (PP) test is used to determine the stationarity of time series data for each feature in a
|
25
|
+
dataset, which is crucial for forecasting tasks. It tests the null hypothesis that a time series is unit-root
|
26
|
+
non-stationary. This is vital for understanding the stochastic behavior of the data and ensuring the robustness and
|
27
|
+
validity of predictions generated by regression analysis models.
|
28
|
+
|
29
|
+
### Test Mechanism
|
30
|
+
|
31
|
+
The PP test is conducted for each feature in the dataset as follows:
|
32
|
+
- A data frame is created from the dataset.
|
33
|
+
- For each column, the Phillips-Perron method calculates the test statistic, p-value, lags used, and number of
|
34
|
+
observations.
|
35
|
+
- The results are then stored for each feature, providing a metric that indicates the stationarity of the time
|
36
|
+
series data.
|
37
|
+
|
38
|
+
### Signs of High Risk
|
39
|
+
|
40
|
+
- A high p-value, indicating that the series has a unit root and is non-stationary.
|
41
|
+
- Test statistic values exceeding critical values, suggesting non-stationarity.
|
42
|
+
- High 'usedlag' value, pointing towards autocorrelation issues that may degrade model performance.
|
43
|
+
|
44
|
+
### Strengths
|
45
|
+
|
46
|
+
- Resilience against heteroskedasticity in the error term.
|
47
|
+
- Effective for long time series data.
|
48
|
+
- Helps in determining whether the time series is stationary, aiding in the selection of suitable forecasting
|
49
|
+
models.
|
50
|
+
|
51
|
+
### Limitations
|
52
|
+
|
53
|
+
- Applicable only within a univariate time series framework.
|
54
|
+
- Relies on asymptotic theory, which may reduce the test’s power for small sample sizes.
|
55
|
+
- Non-stationary time series must be converted to stationary series through differencing, potentially leading to
|
56
|
+
loss of important data points.
|
50
57
|
"""
|
51
58
|
|
52
59
|
name = "phillips_perron"
|
@@ -10,41 +10,49 @@ from validmind.vm_models import Figure, Metric
|
|
10
10
|
|
11
11
|
class RollingStatsPlot(Metric):
|
12
12
|
"""
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
for
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
13
|
+
Evaluates the stationarity of time series data by plotting its rolling mean and standard deviation over a specified
|
14
|
+
window.
|
15
|
+
|
16
|
+
### Purpose
|
17
|
+
|
18
|
+
The `RollingStatsPlot` metric is employed to gauge the stationarity of time series data in a given dataset. This
|
19
|
+
metric specifically evaluates the rolling mean and rolling standard deviation of the dataset over a pre-specified
|
20
|
+
window size. The rolling mean provides an understanding of the average trend in the data, while the rolling
|
21
|
+
standard deviation gauges the volatility of the data within the window. It is critical in preparing time series
|
22
|
+
data for modeling as it reveals key insights into data behavior across time.
|
23
|
+
|
24
|
+
### Test Mechanism
|
25
|
+
|
26
|
+
This mechanism is comprised of two steps. Initially, the rolling mean and standard deviation for each of the
|
27
|
+
dataset's columns are calculated over a window size, which can be user-specified or by default set to 12 data
|
28
|
+
points. Then, the calculated rolling mean and standard deviation are visualized via separate plots, illustrating
|
29
|
+
the trends and volatility in the dataset. A straightforward check is conducted to ensure the existence of columns
|
30
|
+
in the dataset, and to verify that the given dataset has been indexed by its date and time—a necessary prerequisite
|
31
|
+
for time series analysis.
|
32
|
+
|
33
|
+
### Signs of High Risk
|
34
|
+
|
29
35
|
- The presence of non-stationary patterns in either the rolling mean or the rolling standard deviation plots, which
|
30
36
|
could indicate trends or seasonality in the data that may affect the performance of time series models.
|
31
37
|
- Missing columns in the dataset, which would prevent the execution of this metric correctly.
|
32
38
|
- The detection of NaN values in the dataset, which may need to be addressed before the metric can proceed
|
33
39
|
successfully.
|
34
40
|
|
35
|
-
|
36
|
-
|
41
|
+
### Strengths
|
42
|
+
|
43
|
+
- Offers visualizations of trending behavior and volatility within the data, facilitating a broader understanding
|
37
44
|
of the dataset's inherent characteristics.
|
38
|
-
- Checks of the dataset's integrity, such as existence of all required columns and the availability of a
|
39
|
-
index.
|
45
|
+
- Checks of the dataset's integrity, such as the existence of all required columns and the availability of a
|
46
|
+
datetime index.
|
40
47
|
- Adjusts to accommodate various window sizes, thus allowing accurate analysis of data with differing temporal
|
41
48
|
granularities.
|
42
49
|
- Considers each column of the data individually, thereby accommodating multi-feature datasets.
|
43
50
|
|
44
|
-
|
45
|
-
|
51
|
+
### Limitations
|
52
|
+
|
53
|
+
- For all columns, a fixed-size window is utilized. This may not accurately capture patterns in datasets where
|
46
54
|
different features may require different optimal window sizes.
|
47
|
-
- Requires the dataset to be indexed by date and time, hence it may not be
|
55
|
+
- Requires the dataset to be indexed by date and time, hence it may not be usable for datasets without a timestamp
|
48
56
|
index.
|
49
57
|
- Primarily serves for data visualization as it does not facilitate any quantitative measures for stationarity,
|
50
58
|
such as through statistical tests. Therefore, the interpretation is subjective and depends heavily on modeler
|
@@ -5,86 +5,71 @@
|
|
5
5
|
import matplotlib.pyplot as plt
|
6
6
|
import seaborn as sns
|
7
7
|
|
8
|
-
from validmind
|
8
|
+
from validmind import tags, tasks
|
9
9
|
|
10
10
|
|
11
|
-
|
11
|
+
@tags("tabular_data", "visualization")
|
12
|
+
@tasks("classification", "regression")
|
13
|
+
def ScatterPlot(dataset):
|
12
14
|
"""
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
- The
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
-
|
41
|
-
|
42
|
-
|
43
|
-
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
-
|
49
|
-
-
|
50
|
-
|
15
|
+
Assesses visual relationships, patterns, and outliers among features in a dataset through scatter plot matrices.
|
16
|
+
|
17
|
+
### Purpose
|
18
|
+
|
19
|
+
The ScatterPlot test aims to visually analyze a given dataset by constructing a scatter plot matrix of its
|
20
|
+
numerical features. The primary goal is to uncover relationships, patterns, and outliers across different features
|
21
|
+
to provide both quantitative and qualitative insights into multidimensional relationships within the dataset. This
|
22
|
+
visual assessment aids in understanding the efficacy of the chosen features for model training and their
|
23
|
+
suitability.
|
24
|
+
|
25
|
+
### Test Mechanism
|
26
|
+
|
27
|
+
Using the Seaborn library, the ScatterPlot function creates the scatter plot matrix. The process involves
|
28
|
+
retrieving all numerical columns from the dataset and generating a scatter matrix for these columns. The resulting
|
29
|
+
scatter plot provides visual representations of feature relationships. The function also adjusts axis labels for
|
30
|
+
readability and returns the final plot as a Matplotlib Figure object for further analysis and visualization.
|
31
|
+
|
32
|
+
### Signs of High Risk
|
33
|
+
|
34
|
+
- The emergence of non-linear or random patterns across different feature pairs, suggesting complex relationships
|
35
|
+
unsuitable for linear assumptions.
|
36
|
+
- Lack of clear patterns or clusters, indicating weak or non-existent correlations among features, which could
|
37
|
+
challenge certain model types.
|
38
|
+
- Presence of outliers, as visual outliers can adversely influence the model's performance.
|
39
|
+
|
40
|
+
### Strengths
|
41
|
+
|
42
|
+
- Provides insight into the multidimensional relationships among multiple features.
|
43
|
+
- Assists in identifying trends, correlations, and outliers that could affect model performance.
|
44
|
+
- Validates assumptions made during model creation, such as linearity.
|
45
|
+
- Versatile for application in both regression and classification tasks.
|
46
|
+
- Using Seaborn facilitates an intuitive and detailed visual exploration of data.
|
47
|
+
|
48
|
+
### Limitations
|
49
|
+
|
50
|
+
- Scatter plot matrices may become cluttered and hard to decipher as the number of features increases.
|
51
|
+
- Primarily reveals pairwise relationships and may fail to illuminate complex interactions involving three or more
|
52
|
+
features.
|
53
|
+
- Being a visual tool, precision in quantitative analysis might be compromised.
|
54
|
+
- Outliers not clearly visible in plots can be missed, affecting model performance.
|
55
|
+
- Assumes that the dataset can fit into the computer's memory, which might not be valid for extremely large
|
56
|
+
datasets.
|
51
57
|
"""
|
52
58
|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
# rotate y axis labels
|
71
|
-
ax.set_ylabel(ax.get_ylabel(), rotation=45)
|
72
|
-
# set y labels alignment
|
73
|
-
ax.yaxis.get_label().set_horizontalalignment("right")
|
74
|
-
# Get the current figure
|
75
|
-
fig = plt.gcf()
|
76
|
-
|
77
|
-
figures = []
|
78
|
-
figures.append(
|
79
|
-
Figure(
|
80
|
-
for_object=self,
|
81
|
-
key=self.key,
|
82
|
-
figure=fig,
|
83
|
-
)
|
84
|
-
)
|
85
|
-
|
86
|
-
plt.close("all")
|
87
|
-
|
88
|
-
return self.cache_results(
|
89
|
-
figures=figures,
|
90
|
-
)
|
59
|
+
g = sns.pairplot(data=dataset.df, diag_kind="kde")
|
60
|
+
for ax in g.axes.flatten():
|
61
|
+
# rotate x axis labels
|
62
|
+
ax.set_xlabel(ax.get_xlabel(), rotation=45)
|
63
|
+
# rotate y axis labels
|
64
|
+
ax.set_ylabel(ax.get_ylabel(), rotation=45)
|
65
|
+
# set y labels alignment
|
66
|
+
ax.yaxis.get_label().set_horizontalalignment("right")
|
67
|
+
# Get the current figure
|
68
|
+
fig = plt.gcf()
|
69
|
+
|
70
|
+
figures = []
|
71
|
+
figures.append(fig)
|
72
|
+
|
73
|
+
plt.close("all")
|
74
|
+
|
75
|
+
return tuple(figures)
|
@@ -19,41 +19,45 @@ logger = get_logger(__name__)
|
|
19
19
|
|
20
20
|
class SeasonalDecompose(Metric):
|
21
21
|
"""
|
22
|
-
|
23
|
-
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
22
|
+
Assesses patterns and seasonality in a time series dataset by decomposing its features into foundational components.
|
23
|
+
|
24
|
+
### Purpose
|
25
|
+
|
26
|
+
The Seasonal Decompose test aims to decompose the features of a time series dataset into their fundamental
|
27
|
+
components: observed, trend, seasonal, and residuals. By utilizing the Seasonal Decomposition of Time Series by
|
28
|
+
Loess (STL) method, the test identifies underlying patterns, predominantly seasonality, in the dataset's features.
|
29
|
+
This aids in developing a more comprehensive understanding of the dataset, which in turn facilitates more effective
|
30
|
+
model validation.
|
31
|
+
|
32
|
+
### Test Mechanism
|
33
|
+
|
34
|
+
The testing process leverages the `seasonal_decompose` function from the `statsmodels.tsa.seasonal` library to
|
35
|
+
evaluate each feature in the dataset. It isolates each feature into four components—observed, trend, seasonal, and
|
36
|
+
residuals—and generates six subplot graphs per feature for visual interpretation. Prior to decomposition, the test
|
37
|
+
scrutinizes and removes any non-finite values, ensuring the reliability of the analysis.
|
38
|
+
|
39
|
+
### Signs of High Risk
|
40
|
+
|
41
|
+
- **Non-Finiteness**: Datasets with a high number of non-finite values may flag as high risk since these values are
|
38
42
|
omitted before conducting the seasonal decomposition.
|
39
|
-
- **Frequent Warnings**:
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
- **Seasonality Detection**:
|
46
|
-
- **Visualization**:
|
47
|
-
- **Unrestricted Usage**:
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
- **Dependence on Assumptions**:
|
52
|
-
frequency
|
53
|
-
- **Handling Non-
|
54
|
-
|
55
|
-
- **Unreliability with Noisy Datasets**:
|
56
|
-
|
43
|
+
- **Frequent Warnings**: Chronic failure to infer the frequency for a scrutinized feature indicates high risk.
|
44
|
+
- **High Seasonality**: A significant seasonal component could potentially render forecasts unreliable due to
|
45
|
+
overwhelming seasonal variation.
|
46
|
+
|
47
|
+
### Strengths
|
48
|
+
|
49
|
+
- **Seasonality Detection**: Accurately discerns hidden seasonality patterns in dataset features.
|
50
|
+
- **Visualization**: Facilitates interpretation and comprehension through graphical representations.
|
51
|
+
- **Unrestricted Usage**: Not confined to any specific regression model, promoting wide-ranging applicability.
|
52
|
+
|
53
|
+
### Limitations
|
54
|
+
|
55
|
+
- **Dependence on Assumptions**: Assumes that dataset features are periodically distributed. Features with no
|
56
|
+
inferable frequency are excluded from the test.
|
57
|
+
- **Handling Non-Finite Values**: Disregards non-finite values during analysis, potentially resulting in an
|
58
|
+
incomplete understanding of the dataset.
|
59
|
+
- **Unreliability with Noisy Datasets**: Produces unreliable results when used with datasets that contain heavy
|
60
|
+
noise.
|
57
61
|
"""
|
58
62
|
|
59
63
|
name = "seasonal_decompose"
|
@@ -20,43 +20,41 @@ from validmind.vm_models import (
|
|
20
20
|
@dataclass
|
21
21
|
class Skewness(ThresholdTest):
|
22
22
|
"""
|
23
|
-
Evaluates the skewness of numerical data in a
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
test
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
-
|
57
|
-
|
58
|
-
- The risk grading is largely dependent on a subjective threshold, which may result in excessive strictness or
|
59
|
-
leniency depending upon selection. This factor might require expert input and recurrent iterations for refinement.
|
23
|
+
Evaluates the skewness of numerical data in a dataset to check against a defined threshold, aiming to ensure data
|
24
|
+
quality and optimize model performance.
|
25
|
+
|
26
|
+
### Purpose
|
27
|
+
|
28
|
+
The purpose of the Skewness test is to measure the asymmetry in the distribution of data within a predictive
|
29
|
+
machine learning model. Specifically, it evaluates the divergence of said distribution from a normal distribution.
|
30
|
+
Understanding the level of skewness helps identify data quality issues, which are crucial for optimizing the
|
31
|
+
performance of traditional machine learning models in both classification and regression settings.
|
32
|
+
|
33
|
+
### Test Mechanism
|
34
|
+
|
35
|
+
This test calculates the skewness of numerical columns in the dataset, focusing specifically on numerical data
|
36
|
+
types. The calculated skewness value is then compared against a predetermined maximum threshold, which is set by
|
37
|
+
default to 1. If the skewness value is less than this maximum threshold, the test passes; otherwise, it fails. The
|
38
|
+
test results, along with the skewness values and column names, are then recorded for further analysis.
|
39
|
+
|
40
|
+
### Signs of High Risk
|
41
|
+
|
42
|
+
- Substantial skewness levels that significantly exceed the maximum threshold.
|
43
|
+
- Persistent skewness in the data, indicating potential issues with the foundational assumptions of the machine
|
44
|
+
learning model.
|
45
|
+
- Subpar model performance, erroneous predictions, or biased inferences due to skewed data distributions.
|
46
|
+
|
47
|
+
### Strengths
|
48
|
+
|
49
|
+
- Fast and efficient identification of unequal data distributions within a machine learning model.
|
50
|
+
- Adjustable maximum threshold parameter, allowing for customization based on user needs.
|
51
|
+
- Provides a clear quantitative measure to mitigate model risks related to data skewness.
|
52
|
+
|
53
|
+
### Limitations
|
54
|
+
|
55
|
+
- Only evaluates numeric columns, potentially missing skewness or bias in non-numeric data.
|
56
|
+
- Assumes that data should follow a normal distribution, which may not always be applicable to real-world data.
|
57
|
+
- Subjective threshold for risk grading, requiring expert input and recurrent iterations for refinement.
|
60
58
|
"""
|
61
59
|
|
62
60
|
name = "skewness"
|
@@ -10,46 +10,46 @@ from validmind.vm_models import Figure, Metric
|
|
10
10
|
|
11
11
|
class SpreadPlot(Metric):
|
12
12
|
"""
|
13
|
-
|
14
|
-
|
13
|
+
Assesses potential correlations between pairs of time series variables through visualization to enhance
|
14
|
+
understanding of their relationships.
|
15
15
|
|
16
|
-
|
17
|
-
The SpreadPlot metric is intended to graphically illustrate and analyse the relationships between pairs of time
|
18
|
-
series variables within a given dataset. This facilitated understanding helps in identifying and assessing
|
19
|
-
potential time series correlations, like cointegration, between the variables.
|
16
|
+
### Purpose
|
20
17
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
This method is iterated for each unique pair of variables in the dataset.
|
18
|
+
The SpreadPlot test aims to graphically illustrate and analyze the relationships between pairs of time series
|
19
|
+
variables within a given dataset. This facilitated understanding helps in identifying and assessing potential time
|
20
|
+
series correlations, such as cointegration, between the variables.
|
25
21
|
|
26
|
-
|
27
|
-
Potential indicators of high risk related to the SpreadPlot metric might include:
|
22
|
+
### Test Mechanism
|
28
23
|
|
29
|
-
|
30
|
-
|
31
|
-
variables
|
24
|
+
The SpreadPlot test computes and represents the spread between each pair of time series variables in the dataset.
|
25
|
+
Specifically, the difference between two variables is calculated and presented as a line graph. This process is
|
26
|
+
iterated for each unique pair of variables in the dataset, allowing for comprehensive visualization of their
|
27
|
+
relationships.
|
28
|
+
|
29
|
+
### Signs of High Risk
|
30
|
+
|
31
|
+
- Large fluctuations in the spread over a given timespan.
|
32
|
+
- Unexpected patterns or trends that may signal potential risks in the underlying correlations between the
|
33
|
+
variables.
|
32
34
|
- Presence of significant missing data or extreme outlier values, which could potentially skew the spread and
|
33
|
-
indicate high risk
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
-
|
39
|
-
-
|
40
|
-
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
-
|
47
|
-
|
48
|
-
-
|
49
|
-
|
50
|
-
-
|
51
|
-
plots
|
52
|
-
- It might not completely capture intricate non-linear relationships between the variables
|
35
|
+
indicate high risk.
|
36
|
+
|
37
|
+
### Strengths
|
38
|
+
|
39
|
+
- Allows for thorough visual examination and interpretation of the correlations between time-series pairs.
|
40
|
+
- Aids in revealing complex relationships like cointegration.
|
41
|
+
- Enhances interpretability by visualizing the relationships, thereby helping in spotting outliers and trends.
|
42
|
+
- Capable of handling numerous variable pairs from the dataset through a versatile and adaptable process.
|
43
|
+
|
44
|
+
### Limitations
|
45
|
+
|
46
|
+
- Primarily serves as a visualization tool and does not offer quantitative measurements or statistics to
|
47
|
+
objectively determine relationships.
|
48
|
+
- Heavily relies on the quality and granularity of the data—missing data or outliers can notably disturb the
|
49
|
+
interpretation of relationships.
|
50
|
+
- Can become inefficient or difficult to interpret with a high number of variables due to the profuse number of
|
51
|
+
plots.
|
52
|
+
- Might not completely capture intricate non-linear relationships between the variables.
|
53
53
|
"""
|
54
54
|
|
55
55
|
name = "spread_plot"
|