validmind 2.7.12__py3-none-any.whl → 2.8.12__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- validmind/__init__.py +58 -10
- validmind/__version__.py +1 -1
- validmind/ai/test_descriptions.py +17 -73
- validmind/api_client.py +18 -1
- validmind/models/r_model.py +5 -1
- validmind/tests/comparison.py +28 -2
- validmind/tests/data_validation/ACFandPACFPlot.py +4 -1
- validmind/tests/data_validation/AutoMA.py +1 -1
- validmind/tests/data_validation/BivariateScatterPlots.py +5 -1
- validmind/tests/data_validation/BoxPierce.py +3 -1
- validmind/tests/data_validation/ClassImbalance.py +1 -1
- validmind/tests/data_validation/DatasetDescription.py +1 -1
- validmind/tests/data_validation/DickeyFullerGLS.py +1 -1
- validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +5 -10
- validmind/tests/data_validation/HighCardinality.py +5 -1
- validmind/tests/data_validation/HighPearsonCorrelation.py +1 -1
- validmind/tests/data_validation/IQROutliersBarPlot.py +5 -3
- validmind/tests/data_validation/IQROutliersTable.py +5 -2
- validmind/tests/data_validation/IsolationForestOutliers.py +5 -4
- validmind/tests/data_validation/JarqueBera.py +2 -2
- validmind/tests/data_validation/LJungBox.py +2 -2
- validmind/tests/data_validation/LaggedCorrelationHeatmap.py +1 -1
- validmind/tests/data_validation/MissingValues.py +14 -10
- validmind/tests/data_validation/MissingValuesBarPlot.py +3 -1
- validmind/tests/data_validation/MutualInformation.py +2 -1
- validmind/tests/data_validation/PearsonCorrelationMatrix.py +1 -1
- validmind/tests/data_validation/ProtectedClassesCombination.py +2 -0
- validmind/tests/data_validation/ProtectedClassesDescription.py +2 -2
- validmind/tests/data_validation/ProtectedClassesDisparity.py +9 -5
- validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py +10 -2
- validmind/tests/data_validation/RollingStatsPlot.py +2 -1
- validmind/tests/data_validation/ScoreBandDefaultRates.py +4 -2
- validmind/tests/data_validation/SeasonalDecompose.py +1 -1
- validmind/tests/data_validation/ShapiroWilk.py +2 -2
- validmind/tests/data_validation/SpreadPlot.py +1 -1
- validmind/tests/data_validation/TabularCategoricalBarPlots.py +1 -1
- validmind/tests/data_validation/TabularDateTimeHistograms.py +1 -1
- validmind/tests/data_validation/TargetRateBarPlots.py +4 -1
- validmind/tests/data_validation/TimeSeriesFrequency.py +1 -1
- validmind/tests/data_validation/TimeSeriesOutliers.py +7 -2
- validmind/tests/data_validation/WOEBinPlots.py +1 -1
- validmind/tests/data_validation/WOEBinTable.py +1 -1
- validmind/tests/data_validation/ZivotAndrewsArch.py +5 -2
- validmind/tests/data_validation/nlp/CommonWords.py +1 -1
- validmind/tests/data_validation/nlp/Hashtags.py +1 -1
- validmind/tests/data_validation/nlp/LanguageDetection.py +1 -1
- validmind/tests/data_validation/nlp/Mentions.py +1 -1
- validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +5 -1
- validmind/tests/data_validation/nlp/Punctuations.py +1 -1
- validmind/tests/data_validation/nlp/Sentiment.py +3 -1
- validmind/tests/data_validation/nlp/TextDescription.py +1 -1
- validmind/tests/data_validation/nlp/Toxicity.py +1 -1
- validmind/tests/model_validation/BertScore.py +7 -1
- validmind/tests/model_validation/BleuScore.py +7 -1
- validmind/tests/model_validation/ClusterSizeDistribution.py +3 -1
- validmind/tests/model_validation/ContextualRecall.py +9 -1
- validmind/tests/model_validation/FeaturesAUC.py +1 -1
- validmind/tests/model_validation/MeteorScore.py +7 -1
- validmind/tests/model_validation/ModelPredictionResiduals.py +5 -1
- validmind/tests/model_validation/RegardScore.py +6 -1
- validmind/tests/model_validation/RegressionResidualsPlot.py +10 -1
- validmind/tests/model_validation/RougeScore.py +3 -1
- validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +2 -0
- validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +10 -2
- validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +6 -2
- validmind/tests/model_validation/TokenDisparity.py +5 -1
- validmind/tests/model_validation/ToxicityScore.py +3 -1
- validmind/tests/model_validation/embeddings/ClusterDistribution.py +1 -1
- validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +5 -1
- validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +5 -1
- validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +5 -1
- validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +2 -0
- validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +5 -1
- validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +6 -2
- validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +3 -1
- validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +4 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +5 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +5 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +5 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +5 -1
- validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +6 -1
- validmind/tests/model_validation/embeddings/utils.py +6 -9
- validmind/tests/model_validation/ragas/AnswerCorrectness.py +1 -1
- validmind/tests/model_validation/ragas/AspectCritic.py +4 -1
- validmind/tests/model_validation/ragas/ContextEntityRecall.py +1 -1
- validmind/tests/model_validation/ragas/ContextPrecision.py +1 -1
- validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py +1 -1
- validmind/tests/model_validation/ragas/ContextRecall.py +1 -1
- validmind/tests/model_validation/ragas/Faithfulness.py +1 -1
- validmind/tests/model_validation/ragas/NoiseSensitivity.py +1 -1
- validmind/tests/model_validation/ragas/ResponseRelevancy.py +1 -1
- validmind/tests/model_validation/ragas/SemanticSimilarity.py +1 -1
- validmind/tests/model_validation/ragas/utils.py +8 -7
- validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +9 -9
- validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +9 -9
- validmind/tests/model_validation/sklearn/CalibrationCurve.py +5 -2
- validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py +15 -2
- validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +5 -1
- validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +24 -14
- validmind/tests/model_validation/sklearn/CompletenessScore.py +8 -9
- validmind/tests/model_validation/sklearn/ConfusionMatrix.py +22 -3
- validmind/tests/model_validation/sklearn/FeatureImportance.py +6 -2
- validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +12 -9
- validmind/tests/model_validation/sklearn/HomogeneityScore.py +14 -9
- validmind/tests/model_validation/sklearn/HyperParametersTuning.py +4 -2
- validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +6 -1
- validmind/tests/model_validation/sklearn/MinimumAccuracy.py +12 -7
- validmind/tests/model_validation/sklearn/MinimumF1Score.py +12 -7
- validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +18 -7
- validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +8 -2
- validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +5 -1
- validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +5 -1
- validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +6 -1
- validmind/tests/model_validation/sklearn/ROCCurve.py +3 -1
- validmind/tests/model_validation/sklearn/RegressionErrors.py +6 -2
- validmind/tests/model_validation/sklearn/RegressionPerformance.py +13 -8
- validmind/tests/model_validation/sklearn/RegressionR2Square.py +8 -5
- validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +5 -1
- validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py +6 -1
- validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py +10 -2
- validmind/tests/model_validation/sklearn/SilhouettePlot.py +5 -1
- validmind/tests/model_validation/sklearn/VMeasure.py +12 -9
- validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +5 -1
- validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +6 -1
- validmind/tests/model_validation/statsmodels/GINITable.py +8 -1
- validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +2 -2
- validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +6 -2
- validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +8 -2
- validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +3 -1
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +7 -2
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +2 -0
- validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +2 -0
- validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +11 -9
- validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +3 -1
- validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py +11 -1
- validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py +10 -2
- validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py +8 -1
- validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py +18 -2
- validmind/tests/ongoing_monitoring/FeatureDrift.py +9 -2
- validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +8 -2
- validmind/tests/ongoing_monitoring/PredictionCorrelation.py +13 -2
- validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py +13 -2
- validmind/tests/ongoing_monitoring/ROCCurveDrift.py +16 -2
- validmind/tests/ongoing_monitoring/ScoreBandsDrift.py +11 -2
- validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +13 -2
- validmind/tests/prompt_validation/Clarity.py +1 -1
- validmind/tests/prompt_validation/NegativeInstruction.py +1 -1
- validmind/tests/prompt_validation/Robustness.py +6 -1
- validmind/tests/prompt_validation/Specificity.py +1 -1
- validmind/tests/prompt_validation/ai_powered_test.py +5 -4
- validmind/tests/run.py +5 -1
- validmind/utils.py +13 -0
- validmind/vm_models/result/result.py +43 -2
- {validmind-2.7.12.dist-info → validmind-2.8.12.dist-info}/METADATA +3 -2
- {validmind-2.7.12.dist-info → validmind-2.8.12.dist-info}/RECORD +158 -163
- validmind/ai/test_result_description/config.yaml +0 -29
- validmind/ai/test_result_description/context.py +0 -73
- validmind/ai/test_result_description/image_processing.py +0 -124
- validmind/ai/test_result_description/system.jinja +0 -39
- validmind/ai/test_result_description/user.jinja +0 -30
- {validmind-2.7.12.dist-info → validmind-2.8.12.dist-info}/LICENSE +0 -0
- {validmind-2.7.12.dist-info → validmind-2.8.12.dist-info}/WHEEL +0 -0
- {validmind-2.7.12.dist-info → validmind-2.8.12.dist-info}/entry_points.txt +0 -0
@@ -7,7 +7,7 @@ import sys
|
|
7
7
|
|
8
8
|
import pandas as pd
|
9
9
|
|
10
|
-
from validmind import tags, tasks
|
10
|
+
from validmind import RawData, tags, tasks
|
11
11
|
from validmind.errors import MissingDependencyError
|
12
12
|
from validmind.logging import get_logger
|
13
13
|
|
@@ -119,7 +119,7 @@ def ProtectedClassesDisparity(
|
|
119
119
|
mask_significance=True,
|
120
120
|
)
|
121
121
|
|
122
|
-
|
122
|
+
returns = [] # Renamed to 'returns' for clarity
|
123
123
|
for protected_class in protected_classes:
|
124
124
|
plot = ap.disparity(
|
125
125
|
bdf, metrics, protected_class, fairness_threshold=disparity_tolerance
|
@@ -129,12 +129,16 @@ def ProtectedClassesDisparity(
|
|
129
129
|
plot.save(
|
130
130
|
buf, format="png"
|
131
131
|
) # as long as the above library is installed, this will work
|
132
|
-
|
132
|
+
returns.append(buf.getvalue())
|
133
133
|
|
134
134
|
string = "_disparity"
|
135
135
|
metrics_adj = [x + string for x in metrics]
|
136
136
|
|
137
137
|
table = bdf[["attribute_name", "attribute_value"] + b.list_disparities(bdf)]
|
138
|
-
|
138
|
+
returns.append(aqp.plot_disparity_all(bdf, metrics=metrics_adj))
|
139
139
|
|
140
|
-
return (
|
140
|
+
return (
|
141
|
+
table,
|
142
|
+
*returns,
|
143
|
+
RawData(model=model.input_id, dataset=dataset.input_id, disparity_data=bdf),
|
144
|
+
)
|
@@ -8,7 +8,7 @@ import sys
|
|
8
8
|
import matplotlib.pyplot as plt
|
9
9
|
import pandas as pd
|
10
10
|
|
11
|
-
from validmind import tags, tasks
|
11
|
+
from validmind import RawData, tags, tasks
|
12
12
|
from validmind.errors import MissingDependencyError
|
13
13
|
from validmind.logging import get_logger
|
14
14
|
|
@@ -103,7 +103,15 @@ def ProtectedClassesThresholdOptimizer(
|
|
103
103
|
test_df, target, y_pred_opt, protected_classes
|
104
104
|
)
|
105
105
|
|
106
|
-
return
|
106
|
+
return (
|
107
|
+
{"DPR and EOR Table": fairness_metrics.reset_index()},
|
108
|
+
fig,
|
109
|
+
RawData(
|
110
|
+
y_predictions=y_pred_opt.tolist(),
|
111
|
+
dataset=dataset.input_id,
|
112
|
+
protected_classes=protected_classes,
|
113
|
+
),
|
114
|
+
)
|
107
115
|
|
108
116
|
|
109
117
|
def initialize_and_fit_optimizer(pipeline, X_train, y_train, protected_classes_df):
|
@@ -5,7 +5,7 @@
|
|
5
5
|
import numpy as np
|
6
6
|
import pandas as pd
|
7
7
|
|
8
|
-
from validmind import tags, tasks
|
8
|
+
from validmind import RawData, tags, tasks
|
9
9
|
from validmind.vm_models import VMDataset, VMModel
|
10
10
|
|
11
11
|
|
@@ -137,4 +137,6 @@ def ScoreBandDefaultRates(
|
|
137
137
|
}
|
138
138
|
)
|
139
139
|
|
140
|
-
return pd.DataFrame(results)
|
140
|
+
return pd.DataFrame(results), RawData(
|
141
|
+
results=results, model=model.input_id, dataset=dataset.input_id
|
142
|
+
)
|
@@ -166,4 +166,4 @@ def SeasonalDecompose(dataset: VMDataset, seasonal_model: str = "additive"):
|
|
166
166
|
if not figures:
|
167
167
|
raise SkipTestError("No valid features found for seasonal decomposition")
|
168
168
|
|
169
|
-
return (*figures, RawData(decomposed_components=raw_data))
|
169
|
+
return (*figures, RawData(decomposed_components=raw_data, dataset=dataset.input_id))
|
@@ -5,7 +5,7 @@
|
|
5
5
|
import pandas as pd
|
6
6
|
from scipy import stats
|
7
7
|
|
8
|
-
from validmind import tags, tasks
|
8
|
+
from validmind import RawData, tags, tasks
|
9
9
|
|
10
10
|
|
11
11
|
@tasks("classification", "regression")
|
@@ -66,4 +66,4 @@ def ShapiroWilk(dataset):
|
|
66
66
|
sw_df.reset_index(inplace=True)
|
67
67
|
sw_df.columns = ["column", "stat", "pvalue"]
|
68
68
|
|
69
|
-
return sw_df
|
69
|
+
return sw_df, RawData(shapiro_results=sw_values, dataset=dataset.input_id)
|
@@ -5,7 +5,7 @@
|
|
5
5
|
import pandas as pd
|
6
6
|
import plotly.graph_objects as go
|
7
7
|
|
8
|
-
from validmind import tags, tasks
|
8
|
+
from validmind import RawData, tags, tasks
|
9
9
|
from validmind.errors import SkipTestError
|
10
10
|
from validmind.vm_models import VMDataset
|
11
11
|
|
@@ -111,4 +111,9 @@ def TimeSeriesOutliers(dataset: VMDataset, zscore_threshold: int = 3):
|
|
111
111
|
|
112
112
|
figures.append(fig)
|
113
113
|
|
114
|
-
return (
|
114
|
+
return (
|
115
|
+
outlier_df.sort_values(["Column", "Date"]),
|
116
|
+
figures,
|
117
|
+
len(outlier_df) == 0,
|
118
|
+
RawData(outliers=outlier_df, dataset=dataset.input_id),
|
119
|
+
)
|
@@ -6,7 +6,7 @@ import pandas as pd
|
|
6
6
|
from arch.unitroot import ZivotAndrews
|
7
7
|
from numpy.linalg import LinAlgError
|
8
8
|
|
9
|
-
from validmind import tags, tasks
|
9
|
+
from validmind import RawData, tags, tasks
|
10
10
|
from validmind.errors import SkipTestError
|
11
11
|
from validmind.logging import get_logger
|
12
12
|
from validmind.vm_models import VMDataset
|
@@ -83,4 +83,7 @@ def ZivotAndrewsArch(dataset: VMDataset):
|
|
83
83
|
}
|
84
84
|
)
|
85
85
|
|
86
|
-
return
|
86
|
+
return (
|
87
|
+
{"Zivot-Andrews Test Results": za_values},
|
88
|
+
RawData(zivot_andrews=za_values, dataset=dataset.input_id),
|
89
|
+
)
|
@@ -144,4 +144,8 @@ def PolarityAndSubjectivity(dataset, threshold_subjectivity=0.5, threshold_polar
|
|
144
144
|
|
145
145
|
statistics_tables = {"Quadrant Distribution": quadrant_df, "Statistics": stats_df}
|
146
146
|
|
147
|
-
return
|
147
|
+
return (
|
148
|
+
fig,
|
149
|
+
statistics_tables,
|
150
|
+
RawData(sentiment_data=data, dataset=dataset.input_id),
|
151
|
+
)
|
@@ -65,7 +65,7 @@ def Punctuations(dataset, count_mode="token"):
|
|
65
65
|
punctuation_counts = _count_punctuations(corpus, count_mode)
|
66
66
|
fig = _create_punctuation_plot(punctuation_counts)
|
67
67
|
|
68
|
-
return fig, RawData(punctuation_counts=punctuation_counts)
|
68
|
+
return fig, RawData(punctuation_counts=punctuation_counts, dataset=dataset.input_id)
|
69
69
|
|
70
70
|
|
71
71
|
def _create_punctuation_plot(punctuation_counts):
|
@@ -131,4 +131,10 @@ def BertScore(
|
|
131
131
|
# Create a DataFrame from all collected statistics
|
132
132
|
result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
|
133
133
|
|
134
|
-
return (
|
134
|
+
return (
|
135
|
+
result_df,
|
136
|
+
*figures,
|
137
|
+
RawData(
|
138
|
+
bert_scores_df=metrics_df, model=model.input_id, dataset=dataset.input_id
|
139
|
+
),
|
140
|
+
)
|
@@ -114,4 +114,10 @@ def BleuScore(dataset, model):
|
|
114
114
|
# Create a DataFrame from all collected statistics
|
115
115
|
result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
|
116
116
|
|
117
|
-
return (
|
117
|
+
return (
|
118
|
+
result_df,
|
119
|
+
*figures,
|
120
|
+
RawData(
|
121
|
+
bleu_scores_df=metrics_df, model=model.input_id, dataset=dataset.input_id
|
122
|
+
),
|
123
|
+
)
|
@@ -72,4 +72,6 @@ def ClusterSizeDistribution(dataset: VMDataset, model: VMModel):
|
|
72
72
|
fig.update_yaxes(title_text="Counts", showgrid=False)
|
73
73
|
fig.update_layout(title_text="Cluster distribution", title_x=0.5, barmode="group")
|
74
74
|
|
75
|
-
return fig, RawData(
|
75
|
+
return fig, RawData(
|
76
|
+
cluster_counts=df_counts, model=model.input_id, dataset=dataset.input_id
|
77
|
+
)
|
@@ -118,4 +118,12 @@ def ContextualRecall(dataset, model):
|
|
118
118
|
# Create a DataFrame from all collected statistics
|
119
119
|
result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
|
120
120
|
|
121
|
-
return (
|
121
|
+
return (
|
122
|
+
result_df,
|
123
|
+
*figures,
|
124
|
+
RawData(
|
125
|
+
contextual_recall_scores=metrics_df,
|
126
|
+
model=model.input_id,
|
127
|
+
dataset=dataset.input_id,
|
128
|
+
),
|
129
|
+
)
|
@@ -117,4 +117,10 @@ def MeteorScore(dataset, model):
|
|
117
117
|
# Create a DataFrame from all collected statistics
|
118
118
|
result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
|
119
119
|
|
120
|
-
return (
|
120
|
+
return (
|
121
|
+
result_df,
|
122
|
+
*figures,
|
123
|
+
RawData(
|
124
|
+
meteor_scores=metrics_df, model=model.input_id, dataset=dataset.input_id
|
125
|
+
),
|
126
|
+
)
|
@@ -102,4 +102,8 @@ def ModelPredictionResiduals(
|
|
102
102
|
# Create a summary DataFrame for the KS normality test results
|
103
103
|
summary_df = pd.DataFrame([summary])
|
104
104
|
|
105
|
-
return (
|
105
|
+
return (
|
106
|
+
summary_df,
|
107
|
+
*figures,
|
108
|
+
RawData(residuals=residuals, model=model.input_id, dataset=dataset.input_id),
|
109
|
+
)
|
@@ -145,5 +145,10 @@ def RegardScore(dataset, model):
|
|
145
145
|
return (
|
146
146
|
result_df,
|
147
147
|
*figures,
|
148
|
-
RawData(
|
148
|
+
RawData(
|
149
|
+
true_regard=true_df,
|
150
|
+
pred_regard=pred_df,
|
151
|
+
model=model.input_id,
|
152
|
+
dataset=dataset.input_id,
|
153
|
+
),
|
149
154
|
)
|
@@ -105,4 +105,13 @@ def RegressionResidualsPlot(model: VMModel, dataset: VMDataset, bin_size: float
|
|
105
105
|
)
|
106
106
|
)
|
107
107
|
|
108
|
-
return (
|
108
|
+
return (
|
109
|
+
*figures,
|
110
|
+
RawData(
|
111
|
+
residuals=residuals,
|
112
|
+
y_true=y_true,
|
113
|
+
y_pred=y_pred,
|
114
|
+
model=model.input_id,
|
115
|
+
dataset=dataset.input_id,
|
116
|
+
),
|
117
|
+
)
|
@@ -121,5 +121,7 @@ def RougeScore(dataset, model, metric="rouge-1"):
|
|
121
121
|
return (
|
122
122
|
pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"}),
|
123
123
|
*figures,
|
124
|
-
RawData(
|
124
|
+
RawData(
|
125
|
+
rouge_scores_df=df_scores, model=model.input_id, dataset=dataset.input_id
|
126
|
+
),
|
125
127
|
)
|
@@ -4,7 +4,7 @@
|
|
4
4
|
|
5
5
|
import plotly.graph_objects as go
|
6
6
|
|
7
|
-
from validmind import tags, tasks
|
7
|
+
from validmind import RawData, tags, tasks
|
8
8
|
|
9
9
|
|
10
10
|
@tags("model_predictions", "visualization")
|
@@ -70,4 +70,12 @@ def TimeSeriesPredictionsPlot(dataset, model):
|
|
70
70
|
template="plotly_white",
|
71
71
|
)
|
72
72
|
|
73
|
-
|
73
|
+
raw_data = RawData(
|
74
|
+
time_index=time_index,
|
75
|
+
actual_values=dataset.y,
|
76
|
+
predicted_values=y_pred,
|
77
|
+
model=model.input_id,
|
78
|
+
dataset=dataset.input_id,
|
79
|
+
)
|
80
|
+
|
81
|
+
return fig, raw_data
|
@@ -7,7 +7,7 @@ import pandas as pd
|
|
7
7
|
import plotly.express as px
|
8
8
|
from sklearn import metrics
|
9
9
|
|
10
|
-
from validmind import tags, tasks
|
10
|
+
from validmind import RawData, tags, tasks
|
11
11
|
|
12
12
|
|
13
13
|
@tags("model_performance", "sklearn")
|
@@ -105,4 +105,8 @@ def TimeSeriesR2SquareBySegments(dataset, model, segments=None):
|
|
105
105
|
},
|
106
106
|
)
|
107
107
|
|
108
|
-
return
|
108
|
+
return (
|
109
|
+
fig,
|
110
|
+
results_df,
|
111
|
+
RawData(summary=results_df, model=model.input_id, dataset=dataset.input_id),
|
112
|
+
)
|
@@ -108,4 +108,8 @@ def TokenDisparity(dataset, model):
|
|
108
108
|
# Create a DataFrame from all collected statistics
|
109
109
|
result_df = pd.DataFrame(stats_df).reset_index().rename(columns={"index": "Metric"})
|
110
110
|
|
111
|
-
return (
|
111
|
+
return (
|
112
|
+
result_df,
|
113
|
+
*figures,
|
114
|
+
RawData(token_counts_df=df, model=model.input_id, dataset=dataset.input_id),
|
115
|
+
)
|
@@ -141,10 +141,12 @@ def ToxicityScore(dataset, model):
|
|
141
141
|
|
142
142
|
return (
|
143
143
|
result_df,
|
144
|
-
*
|
144
|
+
*figures,
|
145
145
|
RawData(
|
146
146
|
input_toxicity_df=input_df,
|
147
147
|
true_toxicity_df=true_df,
|
148
148
|
pred_toxicity_df=pred_df,
|
149
|
+
model=model.input_id,
|
150
|
+
dataset=dataset.input_id,
|
149
151
|
),
|
150
152
|
)
|
@@ -113,5 +113,9 @@ def CosineSimilarityComparison(dataset, models):
|
|
113
113
|
return (
|
114
114
|
*figures,
|
115
115
|
stats_df,
|
116
|
-
RawData(
|
116
|
+
RawData(
|
117
|
+
similarity_matrices=pd.DataFrame(similarity_matrices),
|
118
|
+
dataset=dataset.input_id,
|
119
|
+
models=[model.input_id for model in models],
|
120
|
+
),
|
117
121
|
)
|
@@ -59,4 +59,8 @@ def CosineSimilarityDistribution(dataset: VMDataset, model: VMModel):
|
|
59
59
|
nbins=100,
|
60
60
|
title="Cosine Similarity Distribution",
|
61
61
|
labels={"x": "Cosine Similarity"},
|
62
|
-
), RawData(
|
62
|
+
), RawData(
|
63
|
+
similarity_scores=similarity_scores,
|
64
|
+
model=model.input_id,
|
65
|
+
dataset=dataset.input_id,
|
66
|
+
)
|
@@ -89,4 +89,8 @@ def EmbeddingsVisualization2D(
|
|
89
89
|
fig = px.scatter(**scatter_kwargs)
|
90
90
|
fig.update_layout(width=500, height=500)
|
91
91
|
|
92
|
-
return fig, RawData(
|
92
|
+
return fig, RawData(
|
93
|
+
tsne_embeddings=reduced_embeddings,
|
94
|
+
model=model.input_id,
|
95
|
+
dataset=dataset.input_id,
|
96
|
+
)
|
@@ -57,7 +57,7 @@ def EuclideanDistanceComparison(dataset, models):
|
|
57
57
|
figures = []
|
58
58
|
all_stats = []
|
59
59
|
|
60
|
-
distance_matrices =
|
60
|
+
distance_matrices = []
|
61
61
|
|
62
62
|
# Generate all pairs of models for comparison
|
63
63
|
for model_A, model_B in combinations(models, 2):
|
@@ -105,6 +105,10 @@ def EuclideanDistanceComparison(dataset, models):
|
|
105
105
|
stats_df = pd.DataFrame(all_stats)
|
106
106
|
|
107
107
|
# Add raw data to return
|
108
|
-
raw_data = RawData(
|
108
|
+
raw_data = RawData(
|
109
|
+
distance_matrices=pd.DataFrame(distance_matrices),
|
110
|
+
dataset=dataset.input_id,
|
111
|
+
models=[model.input_id for model in models],
|
112
|
+
)
|
109
113
|
|
110
114
|
return (stats_df, *figures, raw_data)
|
@@ -97,4 +97,8 @@ def StabilityAnalysisKeyword(
|
|
97
97
|
mean_similarity_threshold,
|
98
98
|
)
|
99
99
|
|
100
|
-
return results, RawData(
|
100
|
+
return results, RawData(
|
101
|
+
original_perturbed_similarity=raw_data,
|
102
|
+
model=model.input_id,
|
103
|
+
dataset=dataset.input_id,
|
104
|
+
)
|
@@ -151,4 +151,8 @@ def StabilityAnalysisRandomNoise(
|
|
151
151
|
mean_similarity_threshold,
|
152
152
|
)
|
153
153
|
|
154
|
-
return result, RawData(
|
154
|
+
return *result, RawData(
|
155
|
+
original_perturbed_similarity=raw_data,
|
156
|
+
model=model.input_id,
|
157
|
+
dataset=dataset.input_id,
|
158
|
+
)
|
@@ -107,4 +107,8 @@ def StabilityAnalysisSynonyms(
|
|
107
107
|
mean_similarity_threshold,
|
108
108
|
)
|
109
109
|
|
110
|
-
return result, RawData(
|
110
|
+
return *result, RawData(
|
111
|
+
original_perturbed_similarity=raw_data,
|
112
|
+
model=model.input_id,
|
113
|
+
dataset=dataset.input_id,
|
114
|
+
)
|
@@ -134,4 +134,8 @@ def StabilityAnalysisTranslation(
|
|
134
134
|
mean_similarity_threshold,
|
135
135
|
)
|
136
136
|
|
137
|
-
return result, RawData(
|
137
|
+
return *result, RawData(
|
138
|
+
original_perturbed_similarity=raw_data,
|
139
|
+
model=model.input_id,
|
140
|
+
dataset=dataset.input_id,
|
141
|
+
)
|
@@ -110,5 +110,10 @@ def TSNEComponentsPairwisePlots(
|
|
110
110
|
|
111
111
|
return (
|
112
112
|
*figures,
|
113
|
-
RawData(
|
113
|
+
RawData(
|
114
|
+
embeddings_scaled=embeddings_scaled,
|
115
|
+
tsne_results=tsne_results,
|
116
|
+
model=model.input_id,
|
117
|
+
dataset=dataset.input_id,
|
118
|
+
),
|
114
119
|
)
|