validmind 2.8.10__py3-none-any.whl → 2.8.20__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- validmind/__init__.py +6 -5
- validmind/__version__.py +1 -1
- validmind/ai/test_descriptions.py +17 -11
- validmind/ai/utils.py +2 -2
- validmind/api_client.py +75 -32
- validmind/client.py +108 -100
- validmind/client_config.py +3 -3
- validmind/datasets/classification/__init__.py +7 -3
- validmind/datasets/credit_risk/lending_club.py +28 -16
- validmind/datasets/nlp/cnn_dailymail.py +10 -4
- validmind/datasets/regression/__init__.py +22 -5
- validmind/errors.py +17 -7
- validmind/input_registry.py +1 -1
- validmind/logging.py +44 -35
- validmind/models/foundation.py +2 -2
- validmind/models/function.py +10 -3
- validmind/template.py +30 -22
- validmind/test_suites/__init__.py +2 -2
- validmind/tests/_store.py +13 -4
- validmind/tests/comparison.py +65 -33
- validmind/tests/data_validation/ACFandPACFPlot.py +4 -1
- validmind/tests/data_validation/AutoMA.py +1 -1
- validmind/tests/data_validation/BivariateScatterPlots.py +5 -1
- validmind/tests/data_validation/BoxPierce.py +3 -1
- validmind/tests/data_validation/ClassImbalance.py +4 -2
- validmind/tests/data_validation/DatasetDescription.py +3 -24
- validmind/tests/data_validation/DescriptiveStatistics.py +1 -1
- validmind/tests/data_validation/DickeyFullerGLS.py +1 -1
- validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +1 -1
- validmind/tests/data_validation/HighCardinality.py +5 -1
- validmind/tests/data_validation/HighPearsonCorrelation.py +1 -1
- validmind/tests/data_validation/IQROutliersBarPlot.py +5 -3
- validmind/tests/data_validation/IQROutliersTable.py +5 -2
- validmind/tests/data_validation/IsolationForestOutliers.py +5 -4
- validmind/tests/data_validation/JarqueBera.py +2 -2
- validmind/tests/data_validation/LJungBox.py +2 -2
- validmind/tests/data_validation/LaggedCorrelationHeatmap.py +1 -1
- validmind/tests/data_validation/MissingValues.py +14 -10
- validmind/tests/data_validation/MissingValuesBarPlot.py +3 -1
- validmind/tests/data_validation/MutualInformation.py +2 -1
- validmind/tests/data_validation/PearsonCorrelationMatrix.py +1 -1
- validmind/tests/data_validation/ProtectedClassesCombination.py +2 -0
- validmind/tests/data_validation/ProtectedClassesDescription.py +2 -2
- validmind/tests/data_validation/ProtectedClassesDisparity.py +9 -5
- validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py +10 -2
- validmind/tests/data_validation/RollingStatsPlot.py +2 -1
- validmind/tests/data_validation/ScoreBandDefaultRates.py +4 -2
- validmind/tests/data_validation/SeasonalDecompose.py +1 -1
- validmind/tests/data_validation/ShapiroWilk.py +2 -2
- validmind/tests/data_validation/Skewness.py +7 -6
- validmind/tests/data_validation/SpreadPlot.py +1 -1
- validmind/tests/data_validation/TabularCategoricalBarPlots.py +1 -1
- validmind/tests/data_validation/TabularDateTimeHistograms.py +1 -1
- validmind/tests/data_validation/TargetRateBarPlots.py +4 -1
- validmind/tests/data_validation/TimeSeriesFrequency.py +1 -1
- validmind/tests/data_validation/TimeSeriesOutliers.py +7 -2
- validmind/tests/data_validation/WOEBinPlots.py +1 -1
- validmind/tests/data_validation/WOEBinTable.py +1 -1
- validmind/tests/data_validation/ZivotAndrewsArch.py +5 -2
- validmind/tests/data_validation/nlp/CommonWords.py +1 -1
- validmind/tests/data_validation/nlp/Hashtags.py +1 -1
- validmind/tests/data_validation/nlp/LanguageDetection.py +1 -1
- validmind/tests/data_validation/nlp/Mentions.py +1 -1
- validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +5 -1
- validmind/tests/data_validation/nlp/Punctuations.py +1 -1
- validmind/tests/data_validation/nlp/Sentiment.py +3 -1
- validmind/tests/data_validation/nlp/TextDescription.py +1 -1
- validmind/tests/data_validation/nlp/Toxicity.py +1 -1
- validmind/tests/decorator.py +14 -11
- validmind/tests/load.py +38 -24
- validmind/tests/model_validation/BertScore.py +7 -1
- validmind/tests/model_validation/BleuScore.py +7 -1
- validmind/tests/model_validation/ClusterSizeDistribution.py +3 -1
- validmind/tests/model_validation/ContextualRecall.py +9 -1
- validmind/tests/model_validation/FeaturesAUC.py +1 -1
- validmind/tests/model_validation/MeteorScore.py +7 -1
- validmind/tests/model_validation/ModelPredictionResiduals.py +5 -1
- validmind/tests/model_validation/RegardScore.py +6 -1
- validmind/tests/model_validation/RegressionResidualsPlot.py +10 -1
- validmind/tests/model_validation/RougeScore.py +3 -1
- validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +2 -0
- validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +10 -2
- validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +6 -2
- validmind/tests/model_validation/TokenDisparity.py +5 -1
- validmind/tests/model_validation/ToxicityScore.py +2 -0
- validmind/tests/model_validation/embeddings/ClusterDistribution.py +1 -1
- validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +5 -1
- validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +5 -1
- validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +5 -1
- validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +2 -0
- validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +5 -1
- validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +6 -2
- validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +3 -1
- validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +4 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +5 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +5 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +5 -1
- validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +5 -1
- validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +6 -1
- validmind/tests/model_validation/ragas/AnswerCorrectness.py +5 -3
- validmind/tests/model_validation/ragas/AspectCritic.py +4 -1
- validmind/tests/model_validation/ragas/ContextEntityRecall.py +5 -3
- validmind/tests/model_validation/ragas/ContextPrecision.py +5 -3
- validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py +5 -3
- validmind/tests/model_validation/ragas/ContextRecall.py +5 -3
- validmind/tests/model_validation/ragas/Faithfulness.py +5 -3
- validmind/tests/model_validation/ragas/NoiseSensitivity.py +1 -1
- validmind/tests/model_validation/ragas/ResponseRelevancy.py +5 -3
- validmind/tests/model_validation/ragas/SemanticSimilarity.py +5 -3
- validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +9 -9
- validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +9 -9
- validmind/tests/model_validation/sklearn/CalibrationCurve.py +5 -2
- validmind/tests/model_validation/sklearn/ClassifierThresholdOptimization.py +28 -5
- validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +5 -1
- validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +24 -14
- validmind/tests/model_validation/sklearn/CompletenessScore.py +8 -9
- validmind/tests/model_validation/sklearn/ConfusionMatrix.py +22 -3
- validmind/tests/model_validation/sklearn/FeatureImportance.py +6 -2
- validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +12 -9
- validmind/tests/model_validation/sklearn/HomogeneityScore.py +14 -9
- validmind/tests/model_validation/sklearn/HyperParametersTuning.py +4 -2
- validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +6 -1
- validmind/tests/model_validation/sklearn/MinimumAccuracy.py +12 -7
- validmind/tests/model_validation/sklearn/MinimumF1Score.py +12 -7
- validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +21 -6
- validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +11 -3
- validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +5 -1
- validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +5 -1
- validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +6 -1
- validmind/tests/model_validation/sklearn/ROCCurve.py +3 -1
- validmind/tests/model_validation/sklearn/RegressionErrors.py +6 -2
- validmind/tests/model_validation/sklearn/RegressionPerformance.py +13 -8
- validmind/tests/model_validation/sklearn/RegressionR2Square.py +8 -5
- validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +5 -1
- validmind/tests/model_validation/sklearn/SHAPGlobalImportance.py +34 -26
- validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py +10 -2
- validmind/tests/model_validation/sklearn/SilhouettePlot.py +5 -1
- validmind/tests/model_validation/sklearn/VMeasure.py +12 -9
- validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +15 -10
- validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +5 -1
- validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +6 -1
- validmind/tests/model_validation/statsmodels/GINITable.py +8 -1
- validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +2 -2
- validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +6 -2
- validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +8 -2
- validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +3 -1
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +7 -2
- validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +2 -0
- validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +2 -0
- validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +4 -2
- validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +3 -1
- validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py +11 -1
- validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py +10 -2
- validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py +8 -1
- validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py +18 -2
- validmind/tests/ongoing_monitoring/FeatureDrift.py +9 -2
- validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +8 -2
- validmind/tests/ongoing_monitoring/PredictionCorrelation.py +13 -2
- validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py +13 -2
- validmind/tests/ongoing_monitoring/ROCCurveDrift.py +16 -2
- validmind/tests/ongoing_monitoring/ScoreBandsDrift.py +11 -2
- validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +13 -2
- validmind/tests/output.py +66 -11
- validmind/tests/prompt_validation/Clarity.py +1 -1
- validmind/tests/prompt_validation/NegativeInstruction.py +1 -1
- validmind/tests/prompt_validation/Robustness.py +6 -1
- validmind/tests/prompt_validation/Specificity.py +1 -1
- validmind/tests/run.py +28 -14
- validmind/tests/test_providers.py +28 -35
- validmind/tests/utils.py +17 -4
- validmind/unit_metrics/__init__.py +1 -1
- validmind/utils.py +295 -31
- validmind/vm_models/dataset/dataset.py +19 -16
- validmind/vm_models/dataset/utils.py +5 -3
- validmind/vm_models/figure.py +6 -6
- validmind/vm_models/input.py +6 -5
- validmind/vm_models/model.py +5 -5
- validmind/vm_models/result/result.py +122 -43
- validmind/vm_models/result/utils.py +9 -28
- validmind/vm_models/test_suite/__init__.py +5 -0
- validmind/vm_models/test_suite/runner.py +5 -5
- validmind/vm_models/test_suite/summary.py +20 -2
- validmind/vm_models/test_suite/test.py +6 -6
- validmind/vm_models/test_suite/test_suite.py +10 -10
- {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/METADATA +4 -5
- {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/RECORD +189 -188
- {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/WHEEL +1 -1
- {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/LICENSE +0 -0
- {validmind-2.8.10.dist-info → validmind-2.8.20.dist-info}/entry_points.txt +0 -0
@@ -16,7 +16,7 @@ logger = get_logger(__name__)
|
|
16
16
|
|
17
17
|
class TestSuiteTest:
|
18
18
|
"""
|
19
|
-
Wraps a 'Test' in a Test Suite and handles logic and state for that test
|
19
|
+
Wraps a 'Test' in a Test Suite and handles logic and state for that test.
|
20
20
|
"""
|
21
21
|
|
22
22
|
test_id: str
|
@@ -28,10 +28,10 @@ class TestSuiteTest:
|
|
28
28
|
_load_failed: bool = False
|
29
29
|
|
30
30
|
def __init__(self, test_id_or_obj):
|
31
|
-
"""Load the test class from the test
|
31
|
+
"""Load the test class from the test ID.
|
32
32
|
|
33
33
|
Args:
|
34
|
-
test_id_or_obj (str): The test
|
34
|
+
test_id_or_obj (str): The test ID or a dict with test ID and other options.
|
35
35
|
"""
|
36
36
|
if isinstance(test_id_or_obj, str):
|
37
37
|
self.test_id = test_id_or_obj
|
@@ -42,7 +42,7 @@ class TestSuiteTest:
|
|
42
42
|
self.name = test_id_to_name(self.test_id)
|
43
43
|
|
44
44
|
def get_default_config(self):
|
45
|
-
"""Returns the default configuration for the test"""
|
45
|
+
"""Returns the default configuration for the test."""
|
46
46
|
try:
|
47
47
|
test_func = load_test(self.test_id)
|
48
48
|
except LoadTestError as e:
|
@@ -66,7 +66,7 @@ class TestSuiteTest:
|
|
66
66
|
return config
|
67
67
|
|
68
68
|
def run(self, fail_fast: bool = False, config: dict = None):
|
69
|
-
"""Run the test"""
|
69
|
+
"""Run the test."""
|
70
70
|
if self._load_failed:
|
71
71
|
return
|
72
72
|
|
@@ -110,7 +110,7 @@ class TestSuiteTest:
|
|
110
110
|
)
|
111
111
|
|
112
112
|
async def log_async(self):
|
113
|
-
"""Log the result for this test to ValidMind"""
|
113
|
+
"""Log the result for this test to ValidMind."""
|
114
114
|
if not self.result:
|
115
115
|
raise ValueError("Cannot log test result before running the test")
|
116
116
|
|
@@ -21,7 +21,7 @@ TOP_LEVEL_SECTION_ID = "__top_level__"
|
|
21
21
|
@dataclass
|
22
22
|
class TestSuiteSection:
|
23
23
|
"""
|
24
|
-
Represents a section in a test suite - Internal use only
|
24
|
+
Represents a section in a test suite - Internal use only.
|
25
25
|
|
26
26
|
In a test suite definition, tests can be grouped into sections by using a dict
|
27
27
|
instead of a string (Test ID). The dict must have a 'section_id' key and a
|
@@ -49,7 +49,7 @@ class TestSuiteSection:
|
|
49
49
|
description: Optional[str] = None
|
50
50
|
|
51
51
|
def get_default_config(self):
|
52
|
-
"""Returns the default configuration for the test suite section"""
|
52
|
+
"""Returns the default configuration for the test suite section."""
|
53
53
|
# TODO: configuration across sections/tests needs more work
|
54
54
|
section_default_config = {}
|
55
55
|
|
@@ -69,7 +69,7 @@ class TestSuite:
|
|
69
69
|
can be run as a suite against datasets and models. Test Suites can be defined by
|
70
70
|
inheriting from this base class and defining the list of tests as a class variable.
|
71
71
|
|
72
|
-
Tests can be a flat list of strings or may be nested into sections by using a dict
|
72
|
+
Tests can be a flat list of strings or may be nested into sections by using a dict.
|
73
73
|
"""
|
74
74
|
|
75
75
|
suite_id: ClassVar[str]
|
@@ -79,14 +79,14 @@ class TestSuite:
|
|
79
79
|
|
80
80
|
def __post_init__(self):
|
81
81
|
"""
|
82
|
-
Post init hook (runs after dataclass init)
|
82
|
+
Post init hook (runs after dataclass init).
|
83
83
|
"""
|
84
84
|
self.sections = []
|
85
85
|
self._build_sections()
|
86
86
|
|
87
87
|
def _build_sections(self):
|
88
88
|
"""
|
89
|
-
Builds the sections for the test suite
|
89
|
+
Builds the sections for the test suite.
|
90
90
|
"""
|
91
91
|
if not self.tests:
|
92
92
|
raise ValueError("No tests found in test suite")
|
@@ -130,7 +130,7 @@ class TestSuite:
|
|
130
130
|
return self.suite_id.title().replace("_", " ")
|
131
131
|
|
132
132
|
def get_tests(self) -> List[str]:
|
133
|
-
"""Get all test suite test objects from all sections"""
|
133
|
+
"""Get all test suite test objects from all sections."""
|
134
134
|
tests = []
|
135
135
|
|
136
136
|
for section in self.sections:
|
@@ -139,21 +139,21 @@ class TestSuite:
|
|
139
139
|
return tests
|
140
140
|
|
141
141
|
def num_tests(self) -> int:
|
142
|
-
"""Returns the total number of tests in the test suite"""
|
142
|
+
"""Returns the total number of tests in the test suite."""
|
143
143
|
return len(self.get_tests())
|
144
144
|
|
145
145
|
def get_default_config(self) -> dict:
|
146
|
-
"""Returns the default configuration for the test suite
|
146
|
+
"""Returns the default configuration for the test suite.
|
147
147
|
|
148
148
|
Each test in a test suite can accept parameters and those parameters can have
|
149
149
|
default values. Both the parameters and their defaults are set in the test
|
150
150
|
class and a config object can be passed to the test suite's run method to
|
151
151
|
override the defaults. This function returns a dictionary containing the
|
152
152
|
parameters and their default values for every test to allow users to view
|
153
|
-
and set values
|
153
|
+
and set values.
|
154
154
|
|
155
155
|
Returns:
|
156
|
-
dict: A dictionary of test names and their default parameters
|
156
|
+
dict: A dictionary of test names and their default parameters.
|
157
157
|
"""
|
158
158
|
default_config = {}
|
159
159
|
|
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.3
|
2
2
|
Name: validmind
|
3
|
-
Version: 2.8.
|
3
|
+
Version: 2.8.20
|
4
4
|
Summary: ValidMind Library
|
5
5
|
License: Commercial License
|
6
6
|
Author: Andres Rodriguez
|
@@ -35,12 +35,12 @@ Requires-Dist: numba (<0.59.0)
|
|
35
35
|
Requires-Dist: numpy
|
36
36
|
Requires-Dist: openai (>=1)
|
37
37
|
Requires-Dist: pandas (>=1.1,<=2.0.3)
|
38
|
-
Requires-Dist: plotly
|
38
|
+
Requires-Dist: plotly (<6.0.0)
|
39
39
|
Requires-Dist: plotly-express
|
40
40
|
Requires-Dist: polars
|
41
41
|
Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
|
42
42
|
Requires-Dist: python-dotenv
|
43
|
-
Requires-Dist: ragas (>=0.2.3) ; extra == "all" or extra == "llm"
|
43
|
+
Requires-Dist: ragas (>=0.2.3,<=0.2.7) ; extra == "all" or extra == "llm"
|
44
44
|
Requires-Dist: rouge (>=1)
|
45
45
|
Requires-Dist: scikit-learn (<1.6.0)
|
46
46
|
Requires-Dist: scipy
|
@@ -57,7 +57,6 @@ Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "
|
|
57
57
|
Requires-Dist: tqdm
|
58
58
|
Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "huggingface" or extra == "llm"
|
59
59
|
Requires-Dist: xgboost (>=1.5.2,<3)
|
60
|
-
Requires-Dist: ydata-profiling
|
61
60
|
Requires-Dist: yfinance (>=0.2.48,<0.3.0)
|
62
61
|
Description-Content-Type: text/markdown
|
63
62
|
|
@@ -84,7 +83,7 @@ ValidMind helps developers, data scientists and risk and compliance stakeholders
|
|
84
83
|
>
|
85
84
|
> Signing up is FREE — **[Register with ValidMind](https://docs.validmind.ai/guide/configuration/register-with-validmind.html)**
|
86
85
|
|
87
|
-
That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and
|
86
|
+
That's right — you can run tests and log documentation even if you don't have a model available, so go ahead and get started with the [**ValidMind Library**](https://docs.validmind.ai/developer/validmind-library.html)!
|
88
87
|
|
89
88
|
### How do I do more with the ValidMind Library?
|
90
89
|
|