validmind 2.8.10__py3-none-any.whl → 2.8.20__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (189) hide show
  1. validmind/__init__.py +6 -5
  2. validmind/__version__.py +1 -1
  3. validmind/ai/test_descriptions.py +17 -11
  4. validmind/ai/utils.py +2 -2
  5. validmind/api_client.py +75 -32
  6. validmind/client.py +108 -100
  7. validmind/client_config.py +3 -3
  8. validmind/datasets/classification/__init__.py +7 -3
  9. validmind/datasets/credit_risk/lending_club.py +28 -16
  10. validmind/datasets/nlp/cnn_dailymail.py +10 -4
  11. validmind/datasets/regression/__init__.py +22 -5
  12. validmind/errors.py +17 -7
  13. validmind/input_registry.py +1 -1
  14. validmind/logging.py +44 -35
  15. validmind/models/foundation.py +2 -2
  16. validmind/models/function.py +10 -3
  17. validmind/template.py +30 -22
  18. validmind/test_suites/__init__.py +2 -2
  19. validmind/tests/_store.py +13 -4
  20. validmind/tests/comparison.py +65 -33
  21. validmind/tests/data_validation/ACFandPACFPlot.py +4 -1
  22. validmind/tests/data_validation/AutoMA.py +1 -1
  23. validmind/tests/data_validation/BivariateScatterPlots.py +5 -1
  24. validmind/tests/data_validation/BoxPierce.py +3 -1
  25. validmind/tests/data_validation/ClassImbalance.py +4 -2
  26. validmind/tests/data_validation/DatasetDescription.py +3 -24
  27. validmind/tests/data_validation/DescriptiveStatistics.py +1 -1
  28. validmind/tests/data_validation/DickeyFullerGLS.py +1 -1
  29. validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +1 -1
  30. validmind/tests/data_validation/HighCardinality.py +5 -1
  31. validmind/tests/data_validation/HighPearsonCorrelation.py +1 -1
  32. validmind/tests/data_validation/IQROutliersBarPlot.py +5 -3
  33. validmind/tests/data_validation/IQROutliersTable.py +5 -2
  34. validmind/tests/data_validation/IsolationForestOutliers.py +5 -4
  35. validmind/tests/data_validation/JarqueBera.py +2 -2
  36. validmind/tests/data_validation/LJungBox.py +2 -2
  37. validmind/tests/data_validation/LaggedCorrelationHeatmap.py +1 -1
  38. validmind/tests/data_validation/MissingValues.py +14 -10
  39. validmind/tests/data_validation/MissingValuesBarPlot.py +3 -1
  40. validmind/tests/data_validation/MutualInformation.py +2 -1
  41. validmind/tests/data_validation/PearsonCorrelationMatrix.py +1 -1
  42. validmind/tests/data_validation/ProtectedClassesCombination.py +2 -0
  43. validmind/tests/data_validation/ProtectedClassesDescription.py +2 -2
  44. validmind/tests/data_validation/ProtectedClassesDisparity.py +9 -5
  45. validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py +10 -2
  46. validmind/tests/data_validation/RollingStatsPlot.py +2 -1
  47. validmind/tests/data_validation/ScoreBandDefaultRates.py +4 -2
  48. validmind/tests/data_validation/SeasonalDecompose.py +1 -1
  49. validmind/tests/data_validation/ShapiroWilk.py +2 -2
  50. validmind/tests/data_validation/Skewness.py +7 -6
  51. validmind/tests/data_validation/SpreadPlot.py +1 -1
  52. validmind/tests/data_validation/TabularCategoricalBarPlots.py +1 -1
  53. validmind/tests/data_validation/TabularDateTimeHistograms.py +1 -1
  54. validmind/tests/data_validation/TargetRateBarPlots.py +4 -1
  55. validmind/tests/data_validation/TimeSeriesFrequency.py +1 -1
  56. validmind/tests/data_validation/TimeSeriesOutliers.py +7 -2
  57. validmind/tests/data_validation/WOEBinPlots.py +1 -1
  58. validmind/tests/data_validation/WOEBinTable.py +1 -1
  59. validmind/tests/data_validation/ZivotAndrewsArch.py +5 -2
  60. validmind/tests/data_validation/nlp/CommonWords.py +1 -1
  61. validmind/tests/data_validation/nlp/Hashtags.py +1 -1
  62. validmind/tests/data_validation/nlp/LanguageDetection.py +1 -1
  63. validmind/tests/data_validation/nlp/Mentions.py +1 -1
  64. validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +5 -1
  65. validmind/tests/data_validation/nlp/Punctuations.py +1 -1
  66. validmind/tests/data_validation/nlp/Sentiment.py +3 -1
  67. validmind/tests/data_validation/nlp/TextDescription.py +1 -1
  68. validmind/tests/data_validation/nlp/Toxicity.py +1 -1
  69. validmind/tests/decorator.py +14 -11
  70. validmind/tests/load.py +38 -24
  71. validmind/tests/model_validation/BertScore.py +7 -1
  72. validmind/tests/model_validation/BleuScore.py +7 -1
  73. validmind/tests/model_validation/ClusterSizeDistribution.py +3 -1
  74. validmind/tests/model_validation/ContextualRecall.py +9 -1
  75. validmind/tests/model_validation/FeaturesAUC.py +1 -1
  76. validmind/tests/model_validation/MeteorScore.py +7 -1
  77. validmind/tests/model_validation/ModelPredictionResiduals.py +5 -1
  78. validmind/tests/model_validation/RegardScore.py +6 -1
  79. validmind/tests/model_validation/RegressionResidualsPlot.py +10 -1
  80. validmind/tests/model_validation/RougeScore.py +3 -1
  81. validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +2 -0
  82. validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +10 -2
  83. validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +6 -2
  84. validmind/tests/model_validation/TokenDisparity.py +5 -1
  85. validmind/tests/model_validation/ToxicityScore.py +2 -0
  86. validmind/tests/model_validation/embeddings/ClusterDistribution.py +1 -1
  87. validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +5 -1
  88. validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +5 -1
  89. validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +5 -1
  90. validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +2 -0
  91. validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +5 -1
  92. validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +6 -2
  93. validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +3 -1
  94. validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +4 -1
  95. validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +5 -1
  96. validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +5 -1
  97. validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +5 -1
  98. validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +5 -1
  99. validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +6 -1
  100. validmind/tests/model_validation/ragas/AnswerCorrectness.py +5 -3
  101. validmind/tests/model_validation/ragas/AspectCritic.py +4 -1
  102. validmind/tests/model_validation/ragas/ContextEntityRecall.py +5 -3
  103. validmind/tests/model_validation/ragas/ContextPrecision.py +5 -3
  104. validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py +5 -3
  105. validmind/tests/model_validation/ragas/ContextRecall.py +5 -3
  106. validmind/tests/model_validation/ragas/Faithfulness.py +5 -3
  107. validmind/tests/model_validation/ragas/NoiseSensitivity.py +1 -1
  108. validmind/tests/model_validation/ragas/ResponseRelevancy.py +5 -3
  109. validmind/tests/model_validation/ragas/SemanticSimilarity.py +5 -3
  110. validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +9 -9
  111. validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +9 -9
  112. validmind/tests/model_validation/sklearn/CalibrationCurve.py +5 -2
  113. validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py +28 -5
  114. validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +5 -1
  115. validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +24 -14
  116. validmind/tests/model_validation/sklearn/CompletenessScore.py +8 -9
  117. validmind/tests/model_validation/sklearn/ConfusionMatrix.py +22 -3
  118. validmind/tests/model_validation/sklearn/FeatureImportance.py +6 -2
  119. validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +12 -9
  120. validmind/tests/model_validation/sklearn/HomogeneityScore.py +14 -9
  121. validmind/tests/model_validation/sklearn/HyperParametersTuning.py +4 -2
  122. validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +6 -1
  123. validmind/tests/model_validation/sklearn/MinimumAccuracy.py +12 -7
  124. validmind/tests/model_validation/sklearn/MinimumF1Score.py +12 -7
  125. validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +21 -6
  126. validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +11 -3
  127. validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +5 -1
  128. validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +5 -1
  129. validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +6 -1
  130. validmind/tests/model_validation/sklearn/ROCCurve.py +3 -1
  131. validmind/tests/model_validation/sklearn/RegressionErrors.py +6 -2
  132. validmind/tests/model_validation/sklearn/RegressionPerformance.py +13 -8
  133. validmind/tests/model_validation/sklearn/RegressionR2Square.py +8 -5
  134. validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +5 -1
  135. validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py +34 -26
  136. validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py +10 -2
  137. validmind/tests/model_validation/sklearn/SilhouettePlot.py +5 -1
  138. validmind/tests/model_validation/sklearn/VMeasure.py +12 -9
  139. validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +15 -10
  140. validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +5 -1
  141. validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +6 -1
  142. validmind/tests/model_validation/statsmodels/GINITable.py +8 -1
  143. validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +2 -2
  144. validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +6 -2
  145. validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +8 -2
  146. validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +3 -1
  147. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +7 -2
  148. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +2 -0
  149. validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +2 -0
  150. validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +4 -2
  151. validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +3 -1
  152. validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py +11 -1
  153. validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py +10 -2
  154. validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py +8 -1
  155. validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py +18 -2
  156. validmind/tests/ongoing_monitoring/FeatureDrift.py +9 -2
  157. validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +8 -2
  158. validmind/tests/ongoing_monitoring/PredictionCorrelation.py +13 -2
  159. validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py +13 -2
  160. validmind/tests/ongoing_monitoring/ROCCurveDrift.py +16 -2
  161. validmind/tests/ongoing_monitoring/ScoreBandsDrift.py +11 -2
  162. validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +13 -2
  163. validmind/tests/output.py +66 -11
  164. validmind/tests/prompt_validation/Clarity.py +1 -1
  165. validmind/tests/prompt_validation/NegativeInstruction.py +1 -1
  166. validmind/tests/prompt_validation/Robustness.py +6 -1
  167. validmind/tests/prompt_validation/Specificity.py +1 -1
  168. validmind/tests/run.py +28 -14
  169. validmind/tests/test_providers.py +28 -35
  170. validmind/tests/utils.py +17 -4
  171. validmind/unit_metrics/__init__.py +1 -1
  172. validmind/utils.py +295 -31
  173. validmind/vm_models/dataset/dataset.py +19 -16
  174. validmind/vm_models/dataset/utils.py +5 -3
  175. validmind/vm_models/figure.py +6 -6
  176. validmind/vm_models/input.py +6 -5
  177. validmind/vm_models/model.py +5 -5
  178. validmind/vm_models/result/result.py +122 -43
  179. validmind/vm_models/result/utils.py +9 -28
  180. validmind/vm_models/test_suite/__init__.py +5 -0
  181. validmind/vm_models/test_suite/runner.py +5 -5
  182. validmind/vm_models/test_suite/summary.py +20 -2
  183. validmind/vm_models/test_suite/test.py +6 -6
  184. validmind/vm_models/test_suite/test_suite.py +10 -10
  185. {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/METADATA +4 -5
  186. {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/RECORD +189 -188
  187. {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/WHEEL +1 -1
  188. {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/LICENSE +0 -0
  189. {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/entry_points.txt +0 -0
@@ -16,7 +16,7 @@ logger = get_logger(__name__)
16
16
 
17
17
  class TestSuiteTest:
18
18
  """
19
- Wraps a 'Test' in a Test Suite and handles logic and state for that test
19
+ Wraps a 'Test' in a Test Suite and handles logic and state for that test.
20
20
  """
21
21
 
22
22
  test_id: str
@@ -28,10 +28,10 @@ class TestSuiteTest:
28
28
  _load_failed: bool = False
29
29
 
30
30
  def __init__(self, test_id_or_obj):
31
- """Load the test class from the test id
31
+ """Load the test class from the test ID.
32
32
 
33
33
  Args:
34
- test_id_or_obj (str): The test id or a dict with test id and other options
34
+ test_id_or_obj (str): The test ID or a dict with test ID and other options.
35
35
  """
36
36
  if isinstance(test_id_or_obj, str):
37
37
  self.test_id = test_id_or_obj
@@ -42,7 +42,7 @@ class TestSuiteTest:
42
42
  self.name = test_id_to_name(self.test_id)
43
43
 
44
44
  def get_default_config(self):
45
- """Returns the default configuration for the test"""
45
+ """Returns the default configuration for the test."""
46
46
  try:
47
47
  test_func = load_test(self.test_id)
48
48
  except LoadTestError as e:
@@ -66,7 +66,7 @@ class TestSuiteTest:
66
66
  return config
67
67
 
68
68
  def run(self, fail_fast: bool = False, config: dict = None):
69
- """Run the test"""
69
+ """Run the test."""
70
70
  if self._load_failed:
71
71
  return
72
72
 
@@ -110,7 +110,7 @@ class TestSuiteTest:
110
110
  )
111
111
 
112
112
  async def log_async(self):
113
- """Log the result for this test to ValidMind"""
113
+ """Log the result for this test to ValidMind."""
114
114
  if not self.result:
115
115
  raise ValueError("Cannot log test result before running the test")
116
116
 
@@ -21,7 +21,7 @@ TOP_LEVEL_SECTION_ID = "__top_level__"
21
21
  @dataclass
22
22
  class TestSuiteSection:
23
23
  """
24
- Represents a section in a test suite - Internal use only
24
+ Represents a section in a test suite - Internal use only.
25
25
 
26
26
  In a test suite definition, tests can be grouped into sections by using a dict
27
27
  instead of a string (Test ID). The dict must have a 'section_id' key and a
@@ -49,7 +49,7 @@ class TestSuiteSection:
49
49
  description: Optional[str] = None
50
50
 
51
51
  def get_default_config(self):
52
- """Returns the default configuration for the test suite section"""
52
+ """Returns the default configuration for the test suite section."""
53
53
  # TODO: configuration across sections/tests needs more work
54
54
  section_default_config = {}
55
55
 
@@ -69,7 +69,7 @@ class TestSuite:
69
69
  can be run as a suite against datasets and models. Test Suites can be defined by
70
70
  inheriting from this base class and defining the list of tests as a class variable.
71
71
 
72
- Tests can be a flat list of strings or may be nested into sections by using a dict
72
+ Tests can be a flat list of strings or may be nested into sections by using a dict.
73
73
  """
74
74
 
75
75
  suite_id: ClassVar[str]
@@ -79,14 +79,14 @@ class TestSuite:
79
79
 
80
80
  def __post_init__(self):
81
81
  """
82
- Post init hook (runs after dataclass init)
82
+ Post init hook (runs after dataclass init).
83
83
  """
84
84
  self.sections = []
85
85
  self._build_sections()
86
86
 
87
87
  def _build_sections(self):
88
88
  """
89
- Builds the sections for the test suite
89
+ Builds the sections for the test suite.
90
90
  """
91
91
  if not self.tests:
92
92
  raise ValueError("No tests found in test suite")
@@ -130,7 +130,7 @@ class TestSuite:
130
130
  return self.suite_id.title().replace("_", " ")
131
131
 
132
132
  def get_tests(self) -> List[str]:
133
- """Get all test suite test objects from all sections"""
133
+ """Get all test suite test objects from all sections."""
134
134
  tests = []
135
135
 
136
136
  for section in self.sections:
@@ -139,21 +139,21 @@ class TestSuite:
139
139
  return tests
140
140
 
141
141
  def num_tests(self) -> int:
142
- """Returns the total number of tests in the test suite"""
142
+ """Returns the total number of tests in the test suite."""
143
143
  return len(self.get_tests())
144
144
 
145
145
  def get_default_config(self) -> dict:
146
- """Returns the default configuration for the test suite
146
+ """Returns the default configuration for the test suite.
147
147
 
148
148
  Each test in a test suite can accept parameters and those parameters can have
149
149
  default values. Both the parameters and their defaults are set in the test
150
150
  class and a config object can be passed to the test suite's run method to
151
151
  override the defaults. This function returns a dictionary containing the
152
152
  parameters and their default values for every test to allow users to view
153
- and set values
153
+ and set values.
154
154
 
155
155
  Returns:
156
- dict: A dictionary of test names and their default parameters
156
+ dict: A dictionary of test names and their default parameters.
157
157
  """
158
158
  default_config = {}
159
159
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.3
2
2
  Name: validmind
3
- Version: 2.8.10
3
+ Version: 2.8.20
4
4
  Summary: ValidMind Library
5
5
  License: Commercial License
6
6
  Author: Andres Rodriguez
@@ -35,12 +35,12 @@ Requires-Dist: numba (<0.59.0)
35
35
  Requires-Dist: numpy
36
36
  Requires-Dist: openai (>=1)
37
37
  Requires-Dist: pandas (>=1.1,<=2.0.3)
38
- Requires-Dist: plotly
38
+ Requires-Dist: plotly (<6.0.0)
39
39
  Requires-Dist: plotly-express
40
40
  Requires-Dist: polars
41
41
  Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
42
42
  Requires-Dist: python-dotenv
43
- Requires-Dist: ragas (>=0.2.3) ; extra == "all" or extra == "llm"
43
+ Requires-Dist: ragas (>=0.2.3,<=0.2.7) ; extra == "all" or extra == "llm"
44
44
  Requires-Dist: rouge (>=1)
45
45
  Requires-Dist: scikit-learn (<1.6.0)
46
46
  Requires-Dist: scipy
@@ -57,7 +57,6 @@ Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "
57
57
  Requires-Dist: tqdm
58
58
  Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
59
59
  Requires-Dist: xgboost (>=1.5.2,<3)
60
- Requires-Dist: ydata-profiling
61
60
  Requires-Dist: yfinance (>=0.2.48,<0.3.0)
62
61
  Description-Content-Type: text/markdown
63
62
 
@@ -84,7 +83,7 @@ ValidMind helps developers, data scientists and risk and compliance stakeholders
84
83
  >
85
84
  > Signing up is FREE — **[Register with ValidMind](https://docs.validmind.ai/guide/configuration/register-with-validmind.html)**
86
85
 
87
- That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and [**Get started with the ValidMind Library**](https://docs.validmind.ai/developer/get-started-validmind-library.html)!
86
+ That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and get started with the [**ValidMind Library**](https://docs.validmind.ai/developer/validmind-library.html)!
88
87
 
89
88
  ### How do I do more with the ValidMind Library?
90
89