validmind 2.8.28__py3-none-any.whl → 2.8.29__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (204) hide show
  1. validmind/models/function.py +11 -3
  2. validmind/tests/data_validation/ACFandPACFPlot.py +3 -1
  3. validmind/tests/data_validation/ADF.py +3 -1
  4. validmind/tests/data_validation/AutoAR.py +3 -1
  5. validmind/tests/data_validation/AutoMA.py +5 -1
  6. validmind/tests/data_validation/AutoStationarity.py +5 -1
  7. validmind/tests/data_validation/BivariateScatterPlots.py +3 -1
  8. validmind/tests/data_validation/BoxPierce.py +4 -1
  9. validmind/tests/data_validation/ChiSquaredFeaturesTable.py +1 -1
  10. validmind/tests/data_validation/ClassImbalance.py +1 -1
  11. validmind/tests/data_validation/DatasetDescription.py +4 -1
  12. validmind/tests/data_validation/DatasetSplit.py +3 -2
  13. validmind/tests/data_validation/DescriptiveStatistics.py +3 -1
  14. validmind/tests/data_validation/DickeyFullerGLS.py +3 -1
  15. validmind/tests/data_validation/Duplicates.py +3 -1
  16. validmind/tests/data_validation/EngleGrangerCoint.py +6 -1
  17. validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +3 -1
  18. validmind/tests/data_validation/HighCardinality.py +3 -1
  19. validmind/tests/data_validation/HighPearsonCorrelation.py +4 -1
  20. validmind/tests/data_validation/IQROutliersBarPlot.py +4 -1
  21. validmind/tests/data_validation/IQROutliersTable.py +6 -1
  22. validmind/tests/data_validation/IsolationForestOutliers.py +3 -1
  23. validmind/tests/data_validation/JarqueBera.py +3 -1
  24. validmind/tests/data_validation/KPSS.py +3 -1
  25. validmind/tests/data_validation/LJungBox.py +3 -1
  26. validmind/tests/data_validation/LaggedCorrelationHeatmap.py +6 -1
  27. validmind/tests/data_validation/MissingValues.py +5 -1
  28. validmind/tests/data_validation/MissingValuesBarPlot.py +3 -1
  29. validmind/tests/data_validation/MutualInformation.py +4 -1
  30. validmind/tests/data_validation/PearsonCorrelationMatrix.py +3 -1
  31. validmind/tests/data_validation/PhillipsPerronArch.py +3 -1
  32. validmind/tests/data_validation/ProtectedClassesCombination.py +5 -1
  33. validmind/tests/data_validation/ProtectedClassesDescription.py +5 -1
  34. validmind/tests/data_validation/ProtectedClassesDisparity.py +5 -3
  35. validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py +9 -2
  36. validmind/tests/data_validation/RollingStatsPlot.py +5 -1
  37. validmind/tests/data_validation/RunsTest.py +1 -1
  38. validmind/tests/data_validation/ScatterPlot.py +2 -1
  39. validmind/tests/data_validation/ScoreBandDefaultRates.py +3 -1
  40. validmind/tests/data_validation/SeasonalDecompose.py +6 -1
  41. validmind/tests/data_validation/ShapiroWilk.py +4 -1
  42. validmind/tests/data_validation/Skewness.py +3 -1
  43. validmind/tests/data_validation/SpreadPlot.py +3 -1
  44. validmind/tests/data_validation/TabularCategoricalBarPlots.py +4 -1
  45. validmind/tests/data_validation/TabularDateTimeHistograms.py +3 -1
  46. validmind/tests/data_validation/TabularDescriptionTables.py +4 -1
  47. validmind/tests/data_validation/TabularNumericalHistograms.py +3 -1
  48. validmind/tests/data_validation/TargetRateBarPlots.py +4 -1
  49. validmind/tests/data_validation/TimeSeriesDescription.py +1 -1
  50. validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py +1 -1
  51. validmind/tests/data_validation/TimeSeriesFrequency.py +5 -1
  52. validmind/tests/data_validation/TimeSeriesHistogram.py +4 -1
  53. validmind/tests/data_validation/TimeSeriesLinePlot.py +3 -1
  54. validmind/tests/data_validation/TimeSeriesMissingValues.py +6 -1
  55. validmind/tests/data_validation/TimeSeriesOutliers.py +5 -1
  56. validmind/tests/data_validation/TooManyZeroValues.py +6 -1
  57. validmind/tests/data_validation/UniqueRows.py +5 -1
  58. validmind/tests/data_validation/WOEBinPlots.py +4 -1
  59. validmind/tests/data_validation/WOEBinTable.py +5 -1
  60. validmind/tests/data_validation/ZivotAndrewsArch.py +3 -1
  61. validmind/tests/data_validation/nlp/CommonWords.py +2 -1
  62. validmind/tests/data_validation/nlp/Hashtags.py +2 -1
  63. validmind/tests/data_validation/nlp/LanguageDetection.py +4 -1
  64. validmind/tests/data_validation/nlp/Mentions.py +3 -1
  65. validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +6 -1
  66. validmind/tests/data_validation/nlp/Punctuations.py +2 -1
  67. validmind/tests/data_validation/nlp/Sentiment.py +3 -1
  68. validmind/tests/data_validation/nlp/StopWords.py +2 -1
  69. validmind/tests/data_validation/nlp/TextDescription.py +3 -1
  70. validmind/tests/data_validation/nlp/Toxicity.py +3 -1
  71. validmind/tests/load.py +91 -17
  72. validmind/tests/model_validation/BertScore.py +6 -3
  73. validmind/tests/model_validation/BleuScore.py +6 -1
  74. validmind/tests/model_validation/ClusterSizeDistribution.py +5 -1
  75. validmind/tests/model_validation/ContextualRecall.py +6 -1
  76. validmind/tests/model_validation/FeaturesAUC.py +5 -1
  77. validmind/tests/model_validation/MeteorScore.py +6 -1
  78. validmind/tests/model_validation/ModelMetadata.py +2 -1
  79. validmind/tests/model_validation/ModelPredictionResiduals.py +10 -2
  80. validmind/tests/model_validation/RegardScore.py +7 -1
  81. validmind/tests/model_validation/RegressionResidualsPlot.py +5 -1
  82. validmind/tests/model_validation/RougeScore.py +8 -1
  83. validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +8 -1
  84. validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +7 -1
  85. validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +6 -1
  86. validmind/tests/model_validation/TokenDisparity.py +6 -1
  87. validmind/tests/model_validation/ToxicityScore.py +6 -1
  88. validmind/tests/model_validation/embeddings/ClusterDistribution.py +6 -1
  89. validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +6 -1
  90. validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +6 -1
  91. validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +7 -3
  92. validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +6 -1
  93. validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +4 -3
  94. validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +6 -1
  95. validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +7 -3
  96. validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +6 -1
  97. validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +5 -2
  98. validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +5 -1
  99. validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +4 -1
  100. validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +5 -1
  101. validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +9 -6
  102. validmind/tests/model_validation/ragas/AnswerCorrectness.py +8 -5
  103. validmind/tests/model_validation/ragas/AspectCritic.py +11 -8
  104. validmind/tests/model_validation/ragas/ContextEntityRecall.py +5 -2
  105. validmind/tests/model_validation/ragas/ContextPrecision.py +5 -2
  106. validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py +5 -2
  107. validmind/tests/model_validation/ragas/ContextRecall.py +6 -2
  108. validmind/tests/model_validation/ragas/Faithfulness.py +9 -5
  109. validmind/tests/model_validation/ragas/NoiseSensitivity.py +10 -7
  110. validmind/tests/model_validation/ragas/ResponseRelevancy.py +9 -6
  111. validmind/tests/model_validation/ragas/SemanticSimilarity.py +7 -4
  112. validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +5 -1
  113. validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +5 -1
  114. validmind/tests/model_validation/sklearn/CalibrationCurve.py +5 -1
  115. validmind/tests/model_validation/sklearn/ClassifierPerformance.py +5 -1
  116. validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +5 -1
  117. validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +5 -1
  118. validmind/tests/model_validation/sklearn/CompletenessScore.py +5 -1
  119. validmind/tests/model_validation/sklearn/ConfusionMatrix.py +4 -1
  120. validmind/tests/model_validation/sklearn/FeatureImportance.py +5 -1
  121. validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +5 -1
  122. validmind/tests/model_validation/sklearn/HomogeneityScore.py +5 -1
  123. validmind/tests/model_validation/sklearn/HyperParametersTuning.py +2 -4
  124. validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +3 -3
  125. validmind/tests/model_validation/sklearn/MinimumAccuracy.py +5 -1
  126. validmind/tests/model_validation/sklearn/MinimumF1Score.py +5 -1
  127. validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +5 -1
  128. validmind/tests/model_validation/sklearn/ModelParameters.py +6 -1
  129. validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py +5 -1
  130. validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +3 -2
  131. validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +4 -4
  132. validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +2 -2
  133. validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +5 -1
  134. validmind/tests/model_validation/sklearn/ROCCurve.py +3 -1
  135. validmind/tests/model_validation/sklearn/RegressionErrors.py +6 -1
  136. validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py +6 -1
  137. validmind/tests/model_validation/sklearn/RegressionPerformance.py +5 -1
  138. validmind/tests/model_validation/sklearn/RegressionR2Square.py +6 -1
  139. validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py +6 -1
  140. validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +2 -2
  141. validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py +3 -1
  142. validmind/tests/model_validation/sklearn/SilhouettePlot.py +6 -1
  143. validmind/tests/model_validation/sklearn/TrainingTestDegradation.py +2 -2
  144. validmind/tests/model_validation/sklearn/VMeasure.py +5 -1
  145. validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +6 -5
  146. validmind/tests/model_validation/statsmodels/AutoARIMA.py +3 -1
  147. validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +6 -1
  148. validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +6 -1
  149. validmind/tests/model_validation/statsmodels/GINITable.py +4 -1
  150. validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +5 -1
  151. validmind/tests/model_validation/statsmodels/Lilliefors.py +3 -1
  152. validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +6 -2
  153. validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +4 -1
  154. validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +7 -2
  155. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +5 -4
  156. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +4 -1
  157. validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +3 -2
  158. validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +5 -1
  159. validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +3 -1
  160. validmind/tests/model_validation/statsmodels/ScorecardHistogram.py +6 -1
  161. validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py +2 -2
  162. validmind/tests/ongoing_monitoring/ClassDiscriminationDrift.py +2 -2
  163. validmind/tests/ongoing_monitoring/ClassImbalanceDrift.py +2 -2
  164. validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py +2 -2
  165. validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py +2 -2
  166. validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py +2 -2
  167. validmind/tests/ongoing_monitoring/FeatureDrift.py +5 -2
  168. validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +6 -1
  169. validmind/tests/ongoing_monitoring/PredictionCorrelation.py +8 -1
  170. validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py +2 -2
  171. validmind/tests/ongoing_monitoring/PredictionQuantilesAcrossFeatures.py +6 -1
  172. validmind/tests/ongoing_monitoring/ROCCurveDrift.py +4 -2
  173. validmind/tests/ongoing_monitoring/ScoreBandsDrift.py +2 -2
  174. validmind/tests/ongoing_monitoring/ScorecardHistogramDrift.py +2 -2
  175. validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +8 -1
  176. validmind/tests/prompt_validation/Bias.py +5 -1
  177. validmind/tests/prompt_validation/Clarity.py +5 -1
  178. validmind/tests/prompt_validation/Conciseness.py +5 -1
  179. validmind/tests/prompt_validation/Delimitation.py +5 -1
  180. validmind/tests/prompt_validation/NegativeInstruction.py +5 -1
  181. validmind/tests/prompt_validation/Robustness.py +5 -1
  182. validmind/tests/prompt_validation/Specificity.py +5 -1
  183. validmind/unit_metrics/classification/Accuracy.py +2 -1
  184. validmind/unit_metrics/classification/F1.py +2 -1
  185. validmind/unit_metrics/classification/Precision.py +2 -1
  186. validmind/unit_metrics/classification/ROC_AUC.py +2 -1
  187. validmind/unit_metrics/classification/Recall.py +2 -1
  188. validmind/unit_metrics/regression/AdjustedRSquaredScore.py +2 -1
  189. validmind/unit_metrics/regression/GiniCoefficient.py +2 -1
  190. validmind/unit_metrics/regression/HuberLoss.py +2 -1
  191. validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py +2 -1
  192. validmind/unit_metrics/regression/MeanAbsoluteError.py +2 -1
  193. validmind/unit_metrics/regression/MeanAbsolutePercentageError.py +2 -1
  194. validmind/unit_metrics/regression/MeanBiasDeviation.py +2 -1
  195. validmind/unit_metrics/regression/MeanSquaredError.py +2 -1
  196. validmind/unit_metrics/regression/QuantileLoss.py +1 -1
  197. validmind/unit_metrics/regression/RSquaredScore.py +2 -1
  198. validmind/unit_metrics/regression/RootMeanSquaredError.py +2 -1
  199. validmind/vm_models/dataset/dataset.py +145 -38
  200. {validmind-2.8.28.dist-info → validmind-2.8.29.dist-info}/METADATA +1 -1
  201. {validmind-2.8.28.dist-info → validmind-2.8.29.dist-info}/RECORD +204 -204
  202. {validmind-2.8.28.dist-info → validmind-2.8.29.dist-info}/LICENSE +0 -0
  203. {validmind-2.8.28.dist-info → validmind-2.8.29.dist-info}/WHEEL +0 -0
  204. {validmind-2.8.28.dist-info → validmind-2.8.29.dist-info}/entry_points.txt +0 -0
@@ -3,12 +3,15 @@
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
5
  import warnings
6
+ from typing import Dict, Tuple
6
7
 
7
8
  import plotly.express as px
9
+ import plotly.graph_objects as go
8
10
  from datasets import Dataset
9
11
 
10
12
  from validmind import RawData, tags, tasks
11
13
  from validmind.errors import MissingDependencyError
14
+ from validmind.vm_models import VMDataset
12
15
 
13
16
  from .utils import get_ragas_config, get_renamed_columns
14
17
 
@@ -30,12 +33,12 @@ except ImportError as e:
30
33
  @tags("ragas", "llm")
31
34
  @tasks("text_qa", "text_generation", "text_summarization")
32
35
  def SemanticSimilarity(
33
- dataset,
34
- response_column="response",
35
- reference_column="reference",
36
+ dataset: VMDataset,
37
+ response_column: str = "response",
38
+ reference_column: str = "reference",
36
39
  judge_llm=None,
37
40
  judge_embeddings=None,
38
- ):
41
+ ) -> Tuple[Dict[str, list], go.Figure, go.Figure, RawData]:
39
42
  """
40
43
  Calculates the semantic similarity between generated responses and ground truths
41
44
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  from sklearn.metrics import adjusted_mutual_info_score
6
8
 
7
9
  from validmind import RawData, tags, tasks
@@ -10,7 +12,9 @@ from validmind.vm_models import VMDataset, VMModel
10
12
 
11
13
  @tags("sklearn", "model_performance", "clustering")
12
14
  @tasks("clustering")
13
- def AdjustedMutualInformation(model: VMModel, dataset: VMDataset):
15
+ def AdjustedMutualInformation(
16
+ model: VMModel, dataset: VMDataset
17
+ ) -> Tuple[List[Dict[str, float]], RawData]:
14
18
  """
15
19
  Evaluates clustering model performance by measuring mutual information between true and predicted labels, adjusting
16
20
  for chance.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  from sklearn.metrics import adjusted_rand_score
6
8
 
7
9
  from validmind import RawData, tags, tasks
@@ -10,7 +12,9 @@ from validmind.vm_models import VMDataset, VMModel
10
12
 
11
13
  @tags("sklearn", "model_performance", "clustering")
12
14
  @tasks("clustering")
13
- def AdjustedRandIndex(model: VMModel, dataset: VMDataset):
15
+ def AdjustedRandIndex(
16
+ model: VMModel, dataset: VMDataset
17
+ ) -> Tuple[List[Dict[str, float]], RawData]:
14
18
  """
15
19
  Measures the similarity between two data clusters using the Adjusted Rand Index (ARI) metric in clustering machine
16
20
  learning models.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import plotly.graph_objects as go
6
8
  from sklearn.calibration import calibration_curve
7
9
 
@@ -12,7 +14,9 @@ from validmind.vm_models.result import RawData
12
14
 
13
15
  @tags("sklearn", "model_performance", "classification")
14
16
  @tasks("classification")
15
- def CalibrationCurve(model: VMModel, dataset: VMDataset, n_bins: int = 10):
17
+ def CalibrationCurve(
18
+ model: VMModel, dataset: VMDataset, n_bins: int = 10
19
+ ) -> Tuple[go.Figure, RawData]:
16
20
  """
17
21
  Evaluates the calibration of probability estimates by comparing predicted probabilities against observed
18
22
  frequencies.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List
6
+
5
7
  import numpy as np
6
8
  from sklearn.metrics import classification_report, roc_auc_score
7
9
  from sklearn.preprocessing import LabelBinarizer
@@ -20,7 +22,9 @@ def multiclass_roc_auc_score(y_test, y_pred, average="macro"):
20
22
  "sklearn", "binary_classification", "multiclass_classification", "model_performance"
21
23
  )
22
24
  @tasks("classification", "text_classification")
23
- def ClassifierPerformance(dataset: VMDataset, model: VMModel, average: str = "macro"):
25
+ def ClassifierPerformance(
26
+ dataset: VMDataset, model: VMModel, average: str = "macro"
27
+ ) -> Dict[str, List[Dict[str, float]]]:
24
28
  """
25
29
  Evaluates performance of binary or multiclass classification models using precision, recall, F1-Score, accuracy,
26
30
  and ROC AUC scores.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  import numpy as np
6
8
  from sklearn.metrics.pairwise import cosine_similarity
7
9
 
@@ -12,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
12
14
 
13
15
  @tags("sklearn", "model_performance", "clustering")
14
16
  @tasks("clustering")
15
- def ClusterCosineSimilarity(model: VMModel, dataset: VMDataset):
17
+ def ClusterCosineSimilarity(
18
+ model: VMModel, dataset: VMDataset
19
+ ) -> Tuple[List[Dict[str, float]], RawData]:
16
20
  """
17
21
  Measures the intra-cluster similarity of a clustering model using cosine similarity.
18
22
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  from sklearn.metrics import (
6
8
  adjusted_mutual_info_score,
7
9
  adjusted_rand_score,
@@ -69,7 +71,9 @@ identify members of the same class (precision) and the ability to capture all me
69
71
 
70
72
  @tags("sklearn", "model_performance", "clustering")
71
73
  @tasks("clustering")
72
- def ClusterPerformanceMetrics(model: VMModel, dataset: VMDataset):
74
+ def ClusterPerformanceMetrics(
75
+ model: VMModel, dataset: VMDataset
76
+ ) -> Tuple[List[Dict[str, float]], RawData]:
73
77
  """
74
78
  Evaluates the performance of clustering machine learning models using multiple established metrics.
75
79
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  from sklearn.metrics import completeness_score
6
8
 
7
9
  from validmind import RawData, tags, tasks
@@ -10,7 +12,9 @@ from validmind.vm_models import VMDataset, VMModel
10
12
 
11
13
  @tags("sklearn", "model_performance", "clustering")
12
14
  @tasks("clustering")
13
- def CompletenessScore(model: VMModel, dataset: VMDataset):
15
+ def CompletenessScore(
16
+ model: VMModel, dataset: VMDataset
17
+ ) -> Tuple[List[Dict[str, float]], RawData]:
14
18
  """
15
19
  Evaluates a clustering model's capacity to categorize instances from a single class into the same cluster.
16
20
 
@@ -3,8 +3,11 @@
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
5
 
6
+ from typing import Tuple
7
+
6
8
  import numpy as np
7
9
  import plotly.figure_factory as ff
10
+ import plotly.graph_objects as go
8
11
  from sklearn.metrics import confusion_matrix
9
12
 
10
13
  from validmind import RawData, tags, tasks
@@ -23,7 +26,7 @@ def ConfusionMatrix(
23
26
  dataset: VMDataset,
24
27
  model: VMModel,
25
28
  threshold: float = 0.5,
26
- ):
29
+ ) -> Tuple[go.Figure, RawData]:
27
30
  """
28
31
  Evaluates and visually represents the classification ML model's predictive performance using a Confusion Matrix
29
32
  heatmap.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import pandas as pd
6
8
  from sklearn.inspection import permutation_importance
7
9
 
@@ -11,7 +13,9 @@ from validmind.vm_models import VMDataset, VMModel
11
13
 
12
14
  @tags("model_explainability", "sklearn")
13
15
  @tasks("regression", "time_series_forecasting")
14
- def FeatureImportance(dataset: VMDataset, model: VMModel, num_features: int = 3):
16
+ def FeatureImportance(
17
+ dataset: VMDataset, model: VMModel, num_features: int = 3
18
+ ) -> Tuple[pd.DataFrame, RawData]:
15
19
  """
16
20
  Compute feature importance scores for a given model and generate a summary table
17
21
  with the top important features.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  from sklearn import metrics
6
8
 
7
9
  from validmind import RawData, tags, tasks
@@ -10,7 +12,9 @@ from validmind.vm_models import VMDataset, VMModel
10
12
 
11
13
  @tags("sklearn", "model_performance")
12
14
  @tasks("clustering")
13
- def FowlkesMallowsScore(dataset: VMDataset, model: VMModel):
15
+ def FowlkesMallowsScore(
16
+ dataset: VMDataset, model: VMModel
17
+ ) -> Tuple[List[Dict[str, float]], RawData]:
14
18
  """
15
19
  Evaluates the similarity between predicted and actual cluster assignments in a model using the Fowlkes-Mallows
16
20
  score.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  from sklearn import metrics
6
8
 
7
9
  from validmind import RawData, tags, tasks
@@ -10,7 +12,9 @@ from validmind.vm_models import VMDataset, VMModel
10
12
 
11
13
  @tags("sklearn", "model_performance")
12
14
  @tasks("clustering")
13
- def HomogeneityScore(dataset: VMDataset, model: VMModel):
15
+ def HomogeneityScore(
16
+ dataset: VMDataset, model: VMModel
17
+ ) -> Tuple[List[Dict[str, float]], RawData]:
14
18
  """
15
19
  Assesses clustering homogeneity by comparing true and predicted labels, scoring from 0 (heterogeneous) to 1
16
20
  (homogeneous).
@@ -2,7 +2,7 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
- from typing import Dict, List, Union
5
+ from typing import Dict, List, Tuple, Union
6
6
 
7
7
  from sklearn.metrics import make_scorer, recall_score
8
8
  from sklearn.model_selection import GridSearchCV
@@ -11,8 +11,6 @@ from validmind import RawData, tags, tasks
11
11
  from validmind.vm_models import VMDataset, VMModel
12
12
 
13
13
 
14
- @tags("sklearn", "model_performance")
15
- @tasks("classification", "clustering")
16
14
  def custom_recall(y_true, y_pred_proba, threshold=0.5):
17
15
  y_pred = (y_pred_proba >= threshold).astype(int)
18
16
  return recall_score(y_true, y_pred)
@@ -65,7 +63,7 @@ def HyperParametersTuning(
65
63
  scoring: Union[str, List, Dict] = None,
66
64
  thresholds: Union[float, List[float]] = None,
67
65
  fit_params: dict = None,
68
- ):
66
+ ) -> Tuple[List[Dict[str, float]], RawData]:
69
67
  """
70
68
  Performs exhaustive grid search over specified parameter ranges to find optimal model configurations
71
69
  across different metrics and decision thresholds.
@@ -2,7 +2,7 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
- from typing import List, Union
5
+ from typing import List, Optional, Tuple
6
6
 
7
7
  import numpy as np
8
8
  import plotly.graph_objects as go
@@ -19,8 +19,8 @@ from validmind.vm_models import VMDataset, VMModel
19
19
  @tags("sklearn", "model_performance", "kmeans")
20
20
  @tasks("clustering")
21
21
  def KMeansClustersOptimization(
22
- model: VMModel, dataset: VMDataset, n_clusters: Union[List[int], None] = None
23
- ):
22
+ model: VMModel, dataset: VMDataset, n_clusters: Optional[List[int]] = None
23
+ ) -> Tuple[go.Figure, RawData]:
24
24
  """
25
25
  Optimizes the number of clusters in K-means models using Elbow and Silhouette methods.
26
26
 
@@ -1,6 +1,8 @@
1
1
  # Copyright © 2023-2024 ValidMind Inc. All rights reserved.
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
+ from typing import Dict, List, Tuple
5
+
4
6
  from sklearn.metrics import accuracy_score
5
7
 
6
8
  from validmind import RawData
@@ -12,7 +14,9 @@ from validmind.vm_models import VMDataset, VMModel
12
14
  "sklearn", "binary_classification", "multiclass_classification", "model_performance"
13
15
  )
14
16
  @tasks("classification", "text_classification")
15
- def MinimumAccuracy(dataset: VMDataset, model: VMModel, min_threshold: float = 0.7):
17
+ def MinimumAccuracy(
18
+ dataset: VMDataset, model: VMModel, min_threshold: float = 0.7
19
+ ) -> Tuple[List[Dict[str, float]], bool, RawData]:
16
20
  """
17
21
  Checks if the model's prediction accuracy meets or surpasses a specified threshold.
18
22
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  import numpy as np
6
8
  from sklearn.metrics import f1_score
7
9
 
@@ -14,7 +16,9 @@ from validmind.vm_models import VMDataset, VMModel
14
16
  "sklearn", "binary_classification", "multiclass_classification", "model_performance"
15
17
  )
16
18
  @tasks("classification", "text_classification")
17
- def MinimumF1Score(dataset: VMDataset, model: VMModel, min_threshold: float = 0.5):
19
+ def MinimumF1Score(
20
+ dataset: VMDataset, model: VMModel, min_threshold: float = 0.5
21
+ ) -> Tuple[List[Dict[str, float]], bool, RawData]:
18
22
  """
19
23
  Assesses if the model's F1 score on the validation set meets a predefined minimum threshold, ensuring balanced
20
24
  performance between precision and recall.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  import numpy as np
6
8
  from sklearn.metrics import roc_auc_score
7
9
  from sklearn.preprocessing import LabelBinarizer
@@ -14,7 +16,9 @@ from validmind.vm_models import VMDataset, VMModel
14
16
  "sklearn", "binary_classification", "multiclass_classification", "model_performance"
15
17
  )
16
18
  @tasks("classification", "text_classification")
17
- def MinimumROCAUCScore(dataset: VMDataset, model: VMModel, min_threshold: float = 0.5):
19
+ def MinimumROCAUCScore(
20
+ dataset: VMDataset, model: VMModel, min_threshold: float = 0.5
21
+ ) -> Tuple[List[Dict[str, float]], bool, RawData]:
18
22
  """
19
23
  Validates model by checking if the ROC AUC score meets or surpasses a specified threshold.
20
24
 
@@ -2,14 +2,19 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import List, Optional
6
+
5
7
  import pandas as pd
6
8
 
7
9
  from validmind import tags, tasks
10
+ from validmind.vm_models import VMModel
8
11
 
9
12
 
10
13
  @tags("model_training", "metadata")
11
14
  @tasks("classification", "regression")
12
- def ModelParameters(model, model_params=None):
15
+ def ModelParameters(
16
+ model: VMModel, model_params: Optional[List[str]] = None
17
+ ) -> pd.DataFrame:
13
18
  """
14
19
  Extracts and displays model parameters in a structured format for transparency and reproducibility.
15
20
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List
6
+
5
7
  import numpy as np
6
8
  from sklearn.metrics import classification_report
7
9
 
@@ -19,7 +21,9 @@ from .ClassifierPerformance import multiclass_roc_auc_score
19
21
  "model_comparison",
20
22
  )
21
23
  @tasks("classification", "text_classification")
22
- def ModelsPerformanceComparison(dataset: VMDataset, models: list[VMModel]):
24
+ def ModelsPerformanceComparison(
25
+ dataset: VMDataset, models: list[VMModel]
26
+ ) -> Dict[str, List[Dict[str, float]]]:
23
27
  """
24
28
  Evaluates and compares the performance of multiple Machine Learning models using various metrics like accuracy,
25
29
  precision, recall, and F1 score.
@@ -2,11 +2,12 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
- from typing import List
5
+ from typing import Dict, List, Tuple
6
6
 
7
7
  import matplotlib.pyplot as plt
8
8
  import numpy as np
9
9
  import pandas as pd
10
+ import plotly.graph_objects as go
10
11
  import seaborn as sns
11
12
  from sklearn import metrics
12
13
 
@@ -178,7 +179,7 @@ def OverfitDiagnosis(
178
179
  datasets: List[VMDataset],
179
180
  metric: str = None,
180
181
  cut_off_threshold: float = DEFAULT_THRESHOLD,
181
- ):
182
+ ) -> Tuple[Dict[str, List[Dict[str, float]]], go.Figure, RawData]:
182
183
  """
183
184
  Assesses potential overfitting in a model's predictions, identifying regions where performance between training and
184
185
  testing sets deviates significantly.
@@ -2,7 +2,7 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
- from typing import Union
5
+ from typing import Optional, Tuple
6
6
 
7
7
  import plotly.graph_objects as go
8
8
  from sklearn.inspection import permutation_importance
@@ -26,9 +26,9 @@ logger = get_logger(__name__)
26
26
  def PermutationFeatureImportance(
27
27
  model: VMModel,
28
28
  dataset: VMDataset,
29
- fontsize: Union[int, None] = None,
30
- figure_height: Union[int, None] = None,
31
- ):
29
+ fontsize: Optional[int] = None,
30
+ figure_height: Optional[int] = None,
31
+ ) -> Tuple[go.Figure, RawData]:
32
32
  """
33
33
  Assesses the significance of each feature in a model by evaluating the impact on model performance when feature
34
34
  values are randomly rearranged.
@@ -2,7 +2,7 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
- from typing import List
5
+ from typing import Dict, List, Tuple
6
6
 
7
7
  import numpy as np
8
8
  import pandas as pd
@@ -82,7 +82,7 @@ def calculate_psi(score_initial, score_new, num_bins=10, mode="fixed"):
82
82
  @tasks("classification", "text_classification")
83
83
  def PopulationStabilityIndex(
84
84
  datasets: List[VMDataset], model: VMModel, num_bins: int = 10, mode: str = "fixed"
85
- ):
85
+ ) -> Tuple[Dict[str, List[Dict[str, float]]], go.Figure, RawData]:
86
86
  """
87
87
  Assesses the Population Stability Index (PSI) to quantify the stability of an ML model's predictions across
88
88
  different datasets.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import numpy as np
6
8
  import plotly.graph_objects as go
7
9
  from sklearn.metrics import precision_recall_curve
@@ -14,7 +16,9 @@ from validmind.vm_models import VMDataset, VMModel
14
16
 
15
17
  @tags("sklearn", "binary_classification", "model_performance", "visualization")
16
18
  @tasks("classification", "text_classification")
17
- def PrecisionRecallCurve(model: VMModel, dataset: VMDataset):
19
+ def PrecisionRecallCurve(
20
+ model: VMModel, dataset: VMDataset
21
+ ) -> Tuple[go.Figure, RawData]:
18
22
  """
19
23
  Evaluates the precision-recall trade-off for binary classification models and visualizes the Precision-Recall curve.
20
24
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import numpy as np
6
8
  import plotly.graph_objects as go
7
9
  from sklearn.metrics import roc_auc_score, roc_curve
@@ -19,7 +21,7 @@ from validmind.vm_models import VMDataset, VMModel
19
21
  "visualization",
20
22
  )
21
23
  @tasks("classification", "text_classification")
22
- def ROCCurve(model: VMModel, dataset: VMDataset):
24
+ def ROCCurve(model: VMModel, dataset: VMDataset) -> Tuple[go.Figure, RawData]:
23
25
  """
24
26
  Evaluates binary classification model performance by generating and plotting the Receiver Operating Characteristic
25
27
  (ROC) curve and calculating the Area Under Curve (AUC) score.
@@ -2,16 +2,21 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import numpy as np
6
8
  import pandas as pd
7
9
  from sklearn import metrics
8
10
 
9
11
  from validmind import RawData, tags, tasks
12
+ from validmind.vm_models import VMDataset, VMModel
10
13
 
11
14
 
12
15
  @tags("sklearn", "model_performance")
13
16
  @tasks("regression", "classification")
14
- def RegressionErrors(model, dataset):
17
+ def RegressionErrors(
18
+ model: VMModel, dataset: VMDataset
19
+ ) -> Tuple[pd.DataFrame, RawData]:
15
20
  """
16
21
  Assesses the performance and error distribution of a regression model using various error metrics.
17
22
 
@@ -2,19 +2,24 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import List
6
+
5
7
  import numpy as np
6
8
  import pandas as pd
7
9
  from sklearn import metrics
8
10
 
9
11
  from validmind import tags, tasks
10
12
  from validmind.logging import get_logger
13
+ from validmind.vm_models import VMDataset, VMModel
11
14
 
12
15
  logger = get_logger(__name__)
13
16
 
14
17
 
15
18
  @tags("model_performance", "sklearn")
16
19
  @tasks("regression", "time_series_forecasting")
17
- def RegressionErrorsComparison(datasets, models):
20
+ def RegressionErrorsComparison(
21
+ datasets: List[VMDataset], models: List[VMModel]
22
+ ) -> pd.DataFrame:
18
23
  """
19
24
  Assesses multiple regression error metrics to compare model performance across different datasets, emphasizing
20
25
  systematic overestimation or underestimation and large percentage errors.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, List, Tuple
6
+
5
7
  import numpy as np
6
8
  from sklearn.metrics import mean_absolute_error, mean_squared_error
7
9
 
@@ -14,7 +16,9 @@ logger = get_logger(__name__)
14
16
 
15
17
  @tags("sklearn", "model_performance")
16
18
  @tasks("regression")
17
- def RegressionPerformance(model: VMModel, dataset: VMDataset):
19
+ def RegressionPerformance(
20
+ model: VMModel, dataset: VMDataset
21
+ ) -> Tuple[List[Dict[str, float]], RawData]:
18
22
  """
19
23
  Evaluates the performance of a regression model using five different metrics: MAE, MSE, RMSE, MAPE, and MBD.
20
24
 
@@ -2,16 +2,21 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import pandas as pd
6
8
  from sklearn import metrics
7
9
 
8
10
  from validmind import RawData, tags, tasks
9
11
  from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
12
+ from validmind.vm_models import VMDataset, VMModel
10
13
 
11
14
 
12
15
  @tags("sklearn", "model_performance")
13
16
  @tasks("regression")
14
- def RegressionR2Square(dataset, model):
17
+ def RegressionR2Square(
18
+ dataset: VMDataset, model: VMModel
19
+ ) -> Tuple[pd.DataFrame, RawData]:
15
20
  """
16
21
  Assesses the overall goodness-of-fit of a regression model by evaluating R-squared (R2) and Adjusted R-squared (Adj
17
22
  R2) scores to determine the model's explanatory power over the dependent variable.
@@ -2,16 +2,21 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import List
6
+
5
7
  import pandas as pd
6
8
  from sklearn import metrics
7
9
 
8
10
  from validmind import tags, tasks
9
11
  from validmind.tests.model_validation.statsmodels.statsutils import adj_r2_score
12
+ from validmind.vm_models import VMDataset, VMModel
10
13
 
11
14
 
12
15
  @tags("model_performance", "sklearn")
13
16
  @tasks("regression", "time_series_forecasting")
14
- def RegressionR2SquareComparison(datasets, models):
17
+ def RegressionR2SquareComparison(
18
+ datasets: List[VMDataset], models: List[VMModel]
19
+ ) -> pd.DataFrame:
15
20
  """
16
21
  Compares R-Squared and Adjusted R-Squared values for different regression models across multiple datasets to assess
17
22
  model performance and relevance of features.
@@ -12,9 +12,9 @@ import plotly.graph_objects as go
12
12
  import seaborn as sns
13
13
  from sklearn import metrics
14
14
 
15
+ from validmind import tags, tasks
15
16
  from validmind.errors import MissingOrInvalidModelPredictFnError
16
17
  from validmind.logging import get_logger
17
- from validmind.tests import tags, tasks
18
18
  from validmind.vm_models import VMDataset, VMModel
19
19
 
20
20
  logger = get_logger(__name__)
@@ -221,7 +221,7 @@ def RobustnessDiagnosis(
221
221
  metric: str = None,
222
222
  scaling_factor_std_dev_list: List[float] = DEFAULT_STD_DEV_LIST,
223
223
  performance_decay_threshold: float = DEFAULT_DECAY_THRESHOLD,
224
- ):
224
+ ) -> Tuple[pd.DataFrame, go.Figure, bool]:
225
225
  """
226
226
  Assesses the robustness of a machine learning model by evaluating performance decay under noisy conditions.
227
227