validmind 2.8.28__py3-none-any.whl → 2.9.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (233) hide show
  1. validmind/__version__.py +1 -1
  2. validmind/ai/utils.py +4 -24
  3. validmind/api_client.py +6 -17
  4. validmind/logging.py +48 -0
  5. validmind/models/function.py +11 -3
  6. validmind/tests/__init__.py +2 -0
  7. validmind/tests/__types__.py +18 -0
  8. validmind/tests/data_validation/ACFandPACFPlot.py +3 -1
  9. validmind/tests/data_validation/ADF.py +3 -1
  10. validmind/tests/data_validation/AutoAR.py +3 -1
  11. validmind/tests/data_validation/AutoMA.py +5 -1
  12. validmind/tests/data_validation/AutoStationarity.py +5 -1
  13. validmind/tests/data_validation/BivariateScatterPlots.py +3 -1
  14. validmind/tests/data_validation/BoxPierce.py +4 -1
  15. validmind/tests/data_validation/ChiSquaredFeaturesTable.py +1 -1
  16. validmind/tests/data_validation/ClassImbalance.py +1 -1
  17. validmind/tests/data_validation/DatasetDescription.py +4 -1
  18. validmind/tests/data_validation/DatasetSplit.py +3 -2
  19. validmind/tests/data_validation/DescriptiveStatistics.py +3 -1
  20. validmind/tests/data_validation/DickeyFullerGLS.py +3 -1
  21. validmind/tests/data_validation/Duplicates.py +3 -1
  22. validmind/tests/data_validation/EngleGrangerCoint.py +6 -1
  23. validmind/tests/data_validation/FeatureTargetCorrelationPlot.py +3 -1
  24. validmind/tests/data_validation/HighCardinality.py +3 -1
  25. validmind/tests/data_validation/HighPearsonCorrelation.py +4 -1
  26. validmind/tests/data_validation/IQROutliersBarPlot.py +4 -1
  27. validmind/tests/data_validation/IQROutliersTable.py +6 -1
  28. validmind/tests/data_validation/IsolationForestOutliers.py +3 -1
  29. validmind/tests/data_validation/JarqueBera.py +3 -1
  30. validmind/tests/data_validation/KPSS.py +3 -1
  31. validmind/tests/data_validation/LJungBox.py +3 -1
  32. validmind/tests/data_validation/LaggedCorrelationHeatmap.py +6 -1
  33. validmind/tests/data_validation/MissingValues.py +5 -1
  34. validmind/tests/data_validation/MissingValuesBarPlot.py +3 -1
  35. validmind/tests/data_validation/MutualInformation.py +4 -1
  36. validmind/tests/data_validation/PearsonCorrelationMatrix.py +3 -1
  37. validmind/tests/data_validation/PhillipsPerronArch.py +3 -1
  38. validmind/tests/data_validation/ProtectedClassesCombination.py +5 -1
  39. validmind/tests/data_validation/ProtectedClassesDescription.py +5 -1
  40. validmind/tests/data_validation/ProtectedClassesDisparity.py +5 -3
  41. validmind/tests/data_validation/ProtectedClassesThresholdOptimizer.py +9 -2
  42. validmind/tests/data_validation/RollingStatsPlot.py +5 -1
  43. validmind/tests/data_validation/RunsTest.py +1 -1
  44. validmind/tests/data_validation/ScatterPlot.py +2 -1
  45. validmind/tests/data_validation/ScoreBandDefaultRates.py +3 -1
  46. validmind/tests/data_validation/SeasonalDecompose.py +6 -1
  47. validmind/tests/data_validation/ShapiroWilk.py +4 -1
  48. validmind/tests/data_validation/Skewness.py +3 -1
  49. validmind/tests/data_validation/SpreadPlot.py +3 -1
  50. validmind/tests/data_validation/TabularCategoricalBarPlots.py +4 -1
  51. validmind/tests/data_validation/TabularDateTimeHistograms.py +3 -1
  52. validmind/tests/data_validation/TabularDescriptionTables.py +4 -1
  53. validmind/tests/data_validation/TabularNumericalHistograms.py +3 -1
  54. validmind/tests/data_validation/TargetRateBarPlots.py +4 -1
  55. validmind/tests/data_validation/TimeSeriesDescription.py +1 -1
  56. validmind/tests/data_validation/TimeSeriesDescriptiveStatistics.py +1 -1
  57. validmind/tests/data_validation/TimeSeriesFrequency.py +5 -1
  58. validmind/tests/data_validation/TimeSeriesHistogram.py +4 -1
  59. validmind/tests/data_validation/TimeSeriesLinePlot.py +3 -1
  60. validmind/tests/data_validation/TimeSeriesMissingValues.py +6 -1
  61. validmind/tests/data_validation/TimeSeriesOutliers.py +5 -1
  62. validmind/tests/data_validation/TooManyZeroValues.py +6 -1
  63. validmind/tests/data_validation/UniqueRows.py +5 -1
  64. validmind/tests/data_validation/WOEBinPlots.py +4 -1
  65. validmind/tests/data_validation/WOEBinTable.py +5 -1
  66. validmind/tests/data_validation/ZivotAndrewsArch.py +3 -1
  67. validmind/tests/data_validation/nlp/CommonWords.py +2 -1
  68. validmind/tests/data_validation/nlp/Hashtags.py +2 -1
  69. validmind/tests/data_validation/nlp/LanguageDetection.py +4 -1
  70. validmind/tests/data_validation/nlp/Mentions.py +3 -1
  71. validmind/tests/data_validation/nlp/PolarityAndSubjectivity.py +6 -1
  72. validmind/tests/data_validation/nlp/Punctuations.py +2 -1
  73. validmind/tests/data_validation/nlp/Sentiment.py +3 -1
  74. validmind/tests/data_validation/nlp/StopWords.py +2 -1
  75. validmind/tests/data_validation/nlp/TextDescription.py +3 -1
  76. validmind/tests/data_validation/nlp/Toxicity.py +3 -1
  77. validmind/tests/load.py +91 -17
  78. validmind/tests/model_validation/BertScore.py +6 -3
  79. validmind/tests/model_validation/BleuScore.py +6 -1
  80. validmind/tests/model_validation/ClusterSizeDistribution.py +5 -1
  81. validmind/tests/model_validation/ContextualRecall.py +6 -1
  82. validmind/tests/model_validation/FeaturesAUC.py +5 -1
  83. validmind/tests/model_validation/MeteorScore.py +6 -1
  84. validmind/tests/model_validation/ModelMetadata.py +2 -1
  85. validmind/tests/model_validation/ModelPredictionResiduals.py +10 -2
  86. validmind/tests/model_validation/RegardScore.py +7 -1
  87. validmind/tests/model_validation/RegressionResidualsPlot.py +5 -1
  88. validmind/tests/model_validation/RougeScore.py +8 -1
  89. validmind/tests/model_validation/TimeSeriesPredictionWithCI.py +8 -1
  90. validmind/tests/model_validation/TimeSeriesPredictionsPlot.py +7 -1
  91. validmind/tests/model_validation/TimeSeriesR2SquareBySegments.py +6 -1
  92. validmind/tests/model_validation/TokenDisparity.py +6 -1
  93. validmind/tests/model_validation/ToxicityScore.py +6 -1
  94. validmind/tests/model_validation/embeddings/ClusterDistribution.py +6 -1
  95. validmind/tests/model_validation/embeddings/CosineSimilarityComparison.py +6 -1
  96. validmind/tests/model_validation/embeddings/CosineSimilarityDistribution.py +6 -1
  97. validmind/tests/model_validation/embeddings/CosineSimilarityHeatmap.py +7 -3
  98. validmind/tests/model_validation/embeddings/DescriptiveAnalytics.py +6 -1
  99. validmind/tests/model_validation/embeddings/EmbeddingsVisualization2D.py +4 -3
  100. validmind/tests/model_validation/embeddings/EuclideanDistanceComparison.py +6 -1
  101. validmind/tests/model_validation/embeddings/EuclideanDistanceHeatmap.py +7 -3
  102. validmind/tests/model_validation/embeddings/PCAComponentsPairwisePlots.py +6 -1
  103. validmind/tests/model_validation/embeddings/StabilityAnalysisKeyword.py +5 -2
  104. validmind/tests/model_validation/embeddings/StabilityAnalysisRandomNoise.py +5 -1
  105. validmind/tests/model_validation/embeddings/StabilityAnalysisSynonyms.py +4 -1
  106. validmind/tests/model_validation/embeddings/StabilityAnalysisTranslation.py +5 -1
  107. validmind/tests/model_validation/embeddings/TSNEComponentsPairwisePlots.py +9 -6
  108. validmind/tests/model_validation/ragas/AnswerCorrectness.py +8 -5
  109. validmind/tests/model_validation/ragas/AspectCritic.py +11 -8
  110. validmind/tests/model_validation/ragas/ContextEntityRecall.py +5 -2
  111. validmind/tests/model_validation/ragas/ContextPrecision.py +5 -2
  112. validmind/tests/model_validation/ragas/ContextPrecisionWithoutReference.py +5 -2
  113. validmind/tests/model_validation/ragas/ContextRecall.py +6 -2
  114. validmind/tests/model_validation/ragas/Faithfulness.py +9 -5
  115. validmind/tests/model_validation/ragas/NoiseSensitivity.py +10 -7
  116. validmind/tests/model_validation/ragas/ResponseRelevancy.py +9 -6
  117. validmind/tests/model_validation/ragas/SemanticSimilarity.py +7 -4
  118. validmind/tests/model_validation/sklearn/AdjustedMutualInformation.py +5 -1
  119. validmind/tests/model_validation/sklearn/AdjustedRandIndex.py +5 -1
  120. validmind/tests/model_validation/sklearn/CalibrationCurve.py +5 -1
  121. validmind/tests/model_validation/sklearn/ClassifierPerformance.py +5 -1
  122. validmind/tests/model_validation/sklearn/ClusterCosineSimilarity.py +5 -1
  123. validmind/tests/model_validation/sklearn/ClusterPerformanceMetrics.py +5 -1
  124. validmind/tests/model_validation/sklearn/CompletenessScore.py +5 -1
  125. validmind/tests/model_validation/sklearn/ConfusionMatrix.py +4 -1
  126. validmind/tests/model_validation/sklearn/FeatureImportance.py +5 -1
  127. validmind/tests/model_validation/sklearn/FowlkesMallowsScore.py +5 -1
  128. validmind/tests/model_validation/sklearn/HomogeneityScore.py +5 -1
  129. validmind/tests/model_validation/sklearn/HyperParametersTuning.py +2 -4
  130. validmind/tests/model_validation/sklearn/KMeansClustersOptimization.py +3 -3
  131. validmind/tests/model_validation/sklearn/MinimumAccuracy.py +5 -1
  132. validmind/tests/model_validation/sklearn/MinimumF1Score.py +5 -1
  133. validmind/tests/model_validation/sklearn/MinimumROCAUCScore.py +5 -1
  134. validmind/tests/model_validation/sklearn/ModelParameters.py +6 -1
  135. validmind/tests/model_validation/sklearn/ModelsPerformanceComparison.py +5 -1
  136. validmind/tests/model_validation/sklearn/OverfitDiagnosis.py +3 -2
  137. validmind/tests/model_validation/sklearn/PermutationFeatureImportance.py +4 -4
  138. validmind/tests/model_validation/sklearn/PopulationStabilityIndex.py +2 -2
  139. validmind/tests/model_validation/sklearn/PrecisionRecallCurve.py +5 -1
  140. validmind/tests/model_validation/sklearn/ROCCurve.py +3 -1
  141. validmind/tests/model_validation/sklearn/RegressionErrors.py +6 -1
  142. validmind/tests/model_validation/sklearn/RegressionErrorsComparison.py +6 -1
  143. validmind/tests/model_validation/sklearn/RegressionPerformance.py +5 -1
  144. validmind/tests/model_validation/sklearn/RegressionR2Square.py +6 -1
  145. validmind/tests/model_validation/sklearn/RegressionR2SquareComparison.py +6 -1
  146. validmind/tests/model_validation/sklearn/RobustnessDiagnosis.py +2 -2
  147. validmind/tests/model_validation/sklearn/ScoreProbabilityAlignment.py +3 -1
  148. validmind/tests/model_validation/sklearn/SilhouettePlot.py +6 -1
  149. validmind/tests/model_validation/sklearn/TrainingTestDegradation.py +2 -2
  150. validmind/tests/model_validation/sklearn/VMeasure.py +5 -1
  151. validmind/tests/model_validation/sklearn/WeakspotsDiagnosis.py +6 -5
  152. validmind/tests/model_validation/statsmodels/AutoARIMA.py +3 -1
  153. validmind/tests/model_validation/statsmodels/CumulativePredictionProbabilities.py +6 -1
  154. validmind/tests/model_validation/statsmodels/DurbinWatsonTest.py +6 -1
  155. validmind/tests/model_validation/statsmodels/GINITable.py +4 -1
  156. validmind/tests/model_validation/statsmodels/KolmogorovSmirnov.py +5 -1
  157. validmind/tests/model_validation/statsmodels/Lilliefors.py +3 -1
  158. validmind/tests/model_validation/statsmodels/PredictionProbabilitiesHistogram.py +6 -2
  159. validmind/tests/model_validation/statsmodels/RegressionCoeffs.py +4 -1
  160. validmind/tests/model_validation/statsmodels/RegressionFeatureSignificance.py +7 -2
  161. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlot.py +5 -4
  162. validmind/tests/model_validation/statsmodels/RegressionModelForecastPlotLevels.py +4 -1
  163. validmind/tests/model_validation/statsmodels/RegressionModelSensitivityPlot.py +3 -2
  164. validmind/tests/model_validation/statsmodels/RegressionModelSummary.py +5 -1
  165. validmind/tests/model_validation/statsmodels/RegressionPermutationFeatureImportance.py +3 -1
  166. validmind/tests/model_validation/statsmodels/ScorecardHistogram.py +6 -1
  167. validmind/tests/ongoing_monitoring/CalibrationCurveDrift.py +2 -2
  168. validmind/tests/ongoing_monitoring/ClassDiscriminationDrift.py +2 -2
  169. validmind/tests/ongoing_monitoring/ClassImbalanceDrift.py +2 -2
  170. validmind/tests/ongoing_monitoring/ClassificationAccuracyDrift.py +2 -2
  171. validmind/tests/ongoing_monitoring/ConfusionMatrixDrift.py +2 -2
  172. validmind/tests/ongoing_monitoring/CumulativePredictionProbabilitiesDrift.py +2 -2
  173. validmind/tests/ongoing_monitoring/FeatureDrift.py +5 -2
  174. validmind/tests/ongoing_monitoring/PredictionAcrossEachFeature.py +6 -1
  175. validmind/tests/ongoing_monitoring/PredictionCorrelation.py +8 -1
  176. validmind/tests/ongoing_monitoring/PredictionProbabilitiesHistogramDrift.py +2 -2
  177. validmind/tests/ongoing_monitoring/PredictionQuantilesAcrossFeatures.py +6 -1
  178. validmind/tests/ongoing_monitoring/ROCCurveDrift.py +4 -2
  179. validmind/tests/ongoing_monitoring/ScoreBandsDrift.py +2 -2
  180. validmind/tests/ongoing_monitoring/ScorecardHistogramDrift.py +2 -2
  181. validmind/tests/ongoing_monitoring/TargetPredictionDistributionPlot.py +8 -1
  182. validmind/tests/output.py +9 -2
  183. validmind/tests/plots/BoxPlot.py +260 -0
  184. validmind/tests/plots/CorrelationHeatmap.py +235 -0
  185. validmind/tests/plots/HistogramPlot.py +233 -0
  186. validmind/tests/plots/ViolinPlot.py +125 -0
  187. validmind/tests/plots/__init__.py +0 -0
  188. validmind/tests/prompt_validation/Bias.py +5 -1
  189. validmind/tests/prompt_validation/Clarity.py +5 -1
  190. validmind/tests/prompt_validation/Conciseness.py +5 -1
  191. validmind/tests/prompt_validation/Delimitation.py +5 -1
  192. validmind/tests/prompt_validation/NegativeInstruction.py +5 -1
  193. validmind/tests/prompt_validation/Robustness.py +5 -1
  194. validmind/tests/prompt_validation/Specificity.py +5 -1
  195. validmind/tests/stats/CorrelationAnalysis.py +251 -0
  196. validmind/tests/stats/DescriptiveStats.py +197 -0
  197. validmind/tests/stats/NormalityTests.py +147 -0
  198. validmind/tests/stats/OutlierDetection.py +173 -0
  199. validmind/tests/stats/__init__.py +0 -0
  200. validmind/unit_metrics/classification/Accuracy.py +2 -1
  201. validmind/unit_metrics/classification/F1.py +2 -1
  202. validmind/unit_metrics/classification/Precision.py +2 -1
  203. validmind/unit_metrics/classification/ROC_AUC.py +2 -1
  204. validmind/unit_metrics/classification/Recall.py +2 -1
  205. validmind/unit_metrics/classification/individual/AbsoluteError.py +42 -0
  206. validmind/unit_metrics/classification/individual/BrierScore.py +56 -0
  207. validmind/unit_metrics/classification/individual/CalibrationError.py +77 -0
  208. validmind/unit_metrics/classification/individual/ClassBalance.py +65 -0
  209. validmind/unit_metrics/classification/individual/Confidence.py +52 -0
  210. validmind/unit_metrics/classification/individual/Correctness.py +41 -0
  211. validmind/unit_metrics/classification/individual/LogLoss.py +61 -0
  212. validmind/unit_metrics/classification/individual/OutlierScore.py +86 -0
  213. validmind/unit_metrics/classification/individual/ProbabilityError.py +54 -0
  214. validmind/unit_metrics/classification/individual/Uncertainty.py +60 -0
  215. validmind/unit_metrics/classification/individual/__init__.py +0 -0
  216. validmind/unit_metrics/regression/AdjustedRSquaredScore.py +2 -1
  217. validmind/unit_metrics/regression/GiniCoefficient.py +2 -1
  218. validmind/unit_metrics/regression/HuberLoss.py +2 -1
  219. validmind/unit_metrics/regression/KolmogorovSmirnovStatistic.py +2 -1
  220. validmind/unit_metrics/regression/MeanAbsoluteError.py +2 -1
  221. validmind/unit_metrics/regression/MeanAbsolutePercentageError.py +2 -1
  222. validmind/unit_metrics/regression/MeanBiasDeviation.py +2 -1
  223. validmind/unit_metrics/regression/MeanSquaredError.py +2 -1
  224. validmind/unit_metrics/regression/QuantileLoss.py +1 -1
  225. validmind/unit_metrics/regression/RSquaredScore.py +2 -1
  226. validmind/unit_metrics/regression/RootMeanSquaredError.py +2 -1
  227. validmind/vm_models/dataset/dataset.py +291 -38
  228. validmind/vm_models/result/result.py +26 -4
  229. {validmind-2.8.28.dist-info → validmind-2.9.1.dist-info}/METADATA +2 -2
  230. {validmind-2.8.28.dist-info → validmind-2.9.1.dist-info}/RECORD +233 -212
  231. {validmind-2.8.28.dist-info → validmind-2.9.1.dist-info}/LICENSE +0 -0
  232. {validmind-2.8.28.dist-info → validmind-2.9.1.dist-info}/WHEEL +0 -0
  233. {validmind-2.8.28.dist-info → validmind-2.9.1.dist-info}/entry_points.txt +0 -0
validmind/__version__.py CHANGED
@@ -1 +1 @@
1
- __version__ = "2.8.28"
1
+ __version__ = "2.9.1"
validmind/ai/utils.py CHANGED
@@ -3,9 +3,8 @@
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
5
  import os
6
- from urllib.parse import urljoin
7
6
 
8
- from openai import AzureOpenAI, Client, OpenAI
7
+ from openai import AzureOpenAI, OpenAI
9
8
 
10
9
  from ..logging import get_logger
11
10
  from ..utils import md_to_html
@@ -83,28 +82,9 @@ def get_client_and_model():
83
82
  logger.debug(f"Using Azure OpenAI {__model} for generating descriptions")
84
83
 
85
84
  else:
86
- try:
87
- # TODO: fix circular import
88
- from ..api_client import get_ai_key, get_api_host
89
-
90
- response = get_ai_key()
91
- __client = Client(
92
- base_url=(
93
- # TODO: improve this to be a bit more dynamic
94
- "http://localhost:4000/genai"
95
- if "localhost" in get_api_host()
96
- else urljoin(get_api_host(), "/genai")
97
- ),
98
- api_key=response["key"],
99
- )
100
- __model = "gpt-4o" # TODO: backend should tell us which model to use
101
- logger.debug(f"Using ValidMind {__model} for generating descriptions")
102
- except Exception as e:
103
- logger.debug(f"Failed to get API key: {e}")
104
- raise ValueError(
105
- "OPENAI_API_KEY, AZURE_OPENAI_KEY must be set, or your account "
106
- "must be setup to use ValidMind's LLM in order to use LLM features"
107
- )
85
+ raise ValueError(
86
+ "OPENAI_API_KEY, AZURE_OPENAI_KEY must be setup to use LLM features"
87
+ )
108
88
 
109
89
  return __client, __model
110
90
 
validmind/api_client.py CHANGED
@@ -22,7 +22,7 @@ from ipywidgets import HTML, Accordion
22
22
 
23
23
  from .client_config import client_config
24
24
  from .errors import MissingAPICredentialsError, MissingModelIdError, raise_api_error
25
- from .logging import get_logger, init_sentry, send_single_error
25
+ from .logging import get_logger, init_sentry, log_api_operation, send_single_error
26
26
  from .utils import NumpyEncoder, is_html, md_to_html, run_async
27
27
  from .vm_models import Figure
28
28
 
@@ -85,7 +85,7 @@ def _get_session() -> aiohttp.ClientSession:
85
85
  if not __api_session or __api_session.closed:
86
86
  __api_session = aiohttp.ClientSession(
87
87
  headers=_get_api_headers(),
88
- timeout=aiohttp.ClientTimeout(total=30),
88
+ timeout=aiohttp.ClientTimeout(total=int(os.getenv("VM_API_TIMEOUT", 30))),
89
89
  )
90
90
 
91
91
  return __api_session
@@ -304,6 +304,10 @@ async def alog_metadata(
304
304
  raise e
305
305
 
306
306
 
307
+ @log_api_operation(
308
+ operation_name="Sending figure to ValidMind API",
309
+ extract_key=lambda figure: figure.key,
310
+ )
307
311
  async def alog_figure(figure: Figure) -> Dict[str, Any]:
308
312
  """Logs a figure.
309
313
 
@@ -525,21 +529,6 @@ def log_metric(
525
529
  )
526
530
 
527
531
 
528
- def get_ai_key() -> Dict[str, Any]:
529
- """Calls the API to get an API key for our LLM proxy."""
530
- r = requests.get(
531
- url=_get_url("ai/key"),
532
- headers=_get_api_headers(),
533
- )
534
-
535
- if r.status_code != 200:
536
- # TODO: improve error handling when there's no Open AI API or AI key available
537
- # logger.error("Could not get AI key from ValidMind API")
538
- raise_api_error(r.text)
539
-
540
- return r.json()
541
-
542
-
543
532
  def generate_test_result_description(test_result_data: Dict[str, Any]) -> str:
544
533
  r = requests.post(
545
534
  url=_get_url("ai/generate/test_result_description"),
validmind/logging.py CHANGED
@@ -170,6 +170,54 @@ async def log_performance_async(
170
170
  return wrap
171
171
 
172
172
 
173
+ def log_api_operation(
174
+ operation_name: Optional[str] = None,
175
+ logger: Optional[logging.Logger] = None,
176
+ extract_key: Optional[Callable] = None,
177
+ force: bool = False,
178
+ ) -> Callable[[F], F]:
179
+ """Decorator to log API operations like figure uploads.
180
+
181
+ Args:
182
+ operation_name (str, optional): The name of the operation. Defaults to function name.
183
+ logger (logging.Logger, optional): The logger to use. Defaults to None.
184
+ extract_key (Callable, optional): Function to extract a key from args for logging.
185
+ force (bool, optional): Whether to force logging even if env var is off.
186
+
187
+ Returns:
188
+ Callable: The decorated function.
189
+ """
190
+
191
+ def decorator(func: F) -> F:
192
+ # check if log level is set to debug
193
+ if _get_log_level() != logging.DEBUG and not force:
194
+ return func
195
+
196
+ nonlocal logger
197
+ if logger is None:
198
+ logger = get_logger()
199
+
200
+ nonlocal operation_name
201
+ if operation_name is None:
202
+ operation_name = func.__name__
203
+
204
+ async def wrapped(*args: Any, **kwargs: Any) -> Any:
205
+ # Try to extract a meaningful identifier from the arguments
206
+ identifier = ""
207
+ if extract_key and args:
208
+ try:
209
+ identifier = f": {extract_key(args[0])}"
210
+ except (AttributeError, IndexError):
211
+ pass
212
+
213
+ logger.debug(f"{operation_name}{identifier}")
214
+ return await func(*args, **kwargs)
215
+
216
+ return wrapped
217
+
218
+ return decorator
219
+
220
+
173
221
  def send_single_error(error: Exception) -> None:
174
222
  """Send a single error to Sentry.
175
223
 
@@ -35,7 +35,8 @@ class FunctionModel(VMModel):
35
35
 
36
36
  Attributes:
37
37
  predict_fn (callable): The predict function that should take a dictionary of
38
- input features and return a prediction.
38
+ input features and return a prediction. Can return simple values or
39
+ dictionary objects.
39
40
  input_id (str, optional): The input ID for the model. Defaults to None.
40
41
  name (str, optional): The name of the model. Defaults to the name of the predict_fn.
41
42
  prompt (Prompt, optional): If using a prompt, the prompt object that defines the template
@@ -55,6 +56,13 @@ class FunctionModel(VMModel):
55
56
  X (pandas.DataFrame): The input features to predict on
56
57
 
57
58
  Returns:
58
- List[Any]: The predictions
59
+ List[Any]: The predictions. Can contain simple values or dictionary objects
60
+ depending on what the predict_fn returns.
59
61
  """
60
- return [self.predict_fn(x) for x in X.to_dict(orient="records")]
62
+ predictions = []
63
+ for x in X.to_dict(orient="records"):
64
+ result = self.predict_fn(x)
65
+ # Handle both simple values and complex dictionary returns
66
+ predictions.append(result)
67
+
68
+ return predictions
@@ -43,6 +43,8 @@ __all__ = [
43
43
  "data_validation",
44
44
  "model_validation",
45
45
  "prompt_validation",
46
+ "plots",
47
+ "stats",
46
48
  "list_tests",
47
49
  "load_test",
48
50
  "describe_test",
@@ -187,6 +187,10 @@ TestID = Union[
187
187
  "validmind.ongoing_monitoring.ScoreBandsDrift",
188
188
  "validmind.ongoing_monitoring.ScorecardHistogramDrift",
189
189
  "validmind.ongoing_monitoring.TargetPredictionDistributionPlot",
190
+ "validmind.plots.BoxPlot",
191
+ "validmind.plots.CorrelationHeatmap",
192
+ "validmind.plots.HistogramPlot",
193
+ "validmind.plots.ViolinPlot",
190
194
  "validmind.prompt_validation.Bias",
191
195
  "validmind.prompt_validation.Clarity",
192
196
  "validmind.prompt_validation.Conciseness",
@@ -194,11 +198,25 @@ TestID = Union[
194
198
  "validmind.prompt_validation.NegativeInstruction",
195
199
  "validmind.prompt_validation.Robustness",
196
200
  "validmind.prompt_validation.Specificity",
201
+ "validmind.stats.CorrelationAnalysis",
202
+ "validmind.stats.DescriptiveStats",
203
+ "validmind.stats.NormalityTests",
204
+ "validmind.stats.OutlierDetection",
197
205
  "validmind.unit_metrics.classification.Accuracy",
198
206
  "validmind.unit_metrics.classification.F1",
199
207
  "validmind.unit_metrics.classification.Precision",
200
208
  "validmind.unit_metrics.classification.ROC_AUC",
201
209
  "validmind.unit_metrics.classification.Recall",
210
+ "validmind.unit_metrics.classification.individual.AbsoluteError",
211
+ "validmind.unit_metrics.classification.individual.BrierScore",
212
+ "validmind.unit_metrics.classification.individual.CalibrationError",
213
+ "validmind.unit_metrics.classification.individual.ClassBalance",
214
+ "validmind.unit_metrics.classification.individual.Confidence",
215
+ "validmind.unit_metrics.classification.individual.Correctness",
216
+ "validmind.unit_metrics.classification.individual.LogLoss",
217
+ "validmind.unit_metrics.classification.individual.OutlierScore",
218
+ "validmind.unit_metrics.classification.individual.ProbabilityError",
219
+ "validmind.unit_metrics.classification.individual.Uncertainty",
202
220
  "validmind.unit_metrics.regression.AdjustedRSquaredScore",
203
221
  "validmind.unit_metrics.regression.GiniCoefficient",
204
222
  "validmind.unit_metrics.regression.HuberLoss",
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import pandas as pd
6
8
  import plotly.graph_objects as go
7
9
  from statsmodels.tsa.stattools import acf, pacf
@@ -12,7 +14,7 @@ from validmind.vm_models import VMDataset
12
14
 
13
15
  @tags("time_series_data", "forecasting", "statistical_test", "visualization")
14
16
  @tasks("regression")
15
- def ACFandPACFPlot(dataset: VMDataset):
17
+ def ACFandPACFPlot(dataset: VMDataset) -> Tuple[go.Figure, RawData]:
16
18
  """
17
19
  Analyzes time series data using Autocorrelation Function (ACF) and Partial Autocorrelation Function (PACF) plots to
18
20
  reveal trends and correlations.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict
6
+
5
7
  import pandas as pd
6
8
  from statsmodels.tsa.stattools import adfuller
7
9
 
@@ -16,7 +18,7 @@ logger = get_logger(__name__)
16
18
  "time_series_data", "statsmodels", "forecasting", "statistical_test", "stationarity"
17
19
  )
18
20
  @tasks("regression")
19
- def ADF(dataset: VMDataset):
21
+ def ADF(dataset: VMDataset) -> Dict[str, pd.DataFrame]:
20
22
  """
21
23
  Assesses the stationarity of a time series dataset using the Augmented Dickey-Fuller (ADF) test.
22
24
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict
6
+
5
7
  import pandas as pd
6
8
  from statsmodels.tsa.ar_model import AutoReg
7
9
  from statsmodels.tsa.stattools import adfuller
@@ -15,7 +17,7 @@ logger = get_logger(__name__)
15
17
 
16
18
  @tags("time_series_data", "statsmodels", "forecasting", "statistical_test")
17
19
  @tasks("regression")
18
- def AutoAR(dataset: VMDataset, max_ar_order: int = 3):
20
+ def AutoAR(dataset: VMDataset, max_ar_order: int = 3) -> Dict[str, pd.DataFrame]:
19
21
  """
20
22
  Automatically identifies the optimal Autoregressive (AR) order for a time series using BIC and AIC criteria.
21
23
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict, Tuple
6
+
5
7
  import pandas as pd
6
8
  from statsmodels.tsa.arima.model import ARIMA
7
9
  from statsmodels.tsa.stattools import adfuller
@@ -15,7 +17,9 @@ logger = get_logger(__name__)
15
17
 
16
18
  @tags("time_series_data", "statsmodels", "forecasting", "statistical_test")
17
19
  @tasks("regression")
18
- def AutoMA(dataset: VMDataset, max_ma_order: int = 3):
20
+ def AutoMA(
21
+ dataset: VMDataset, max_ma_order: int = 3
22
+ ) -> Tuple[Dict[str, pd.DataFrame], RawData]:
19
23
  """
20
24
  Automatically selects the optimal Moving Average (MA) order for each variable in a time series dataset based on
21
25
  minimal BIC and AIC values.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Dict
6
+
5
7
  import numpy as np
6
8
  import pandas as pd
7
9
  from statsmodels.tsa.stattools import adfuller
@@ -12,7 +14,9 @@ from validmind.vm_models import VMDataset
12
14
 
13
15
  @tags("time_series_data", "statsmodels", "forecasting", "statistical_test")
14
16
  @tasks("regression")
15
- def AutoStationarity(dataset: VMDataset, max_order: int = 5, threshold: float = 0.05):
17
+ def AutoStationarity(
18
+ dataset: VMDataset, max_order: int = 5, threshold: float = 0.05
19
+ ) -> Dict[str, pd.DataFrame]:
16
20
  """
17
21
  Automates Augmented Dickey-Fuller test to assess stationarity across multiple time series in a DataFrame.
18
22
 
@@ -3,15 +3,17 @@
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
5
  import itertools
6
+ from typing import Tuple
6
7
 
7
8
  import plotly.express as px
9
+ import plotly.graph_objects as go
8
10
 
9
11
  from validmind import RawData, tags, tasks
10
12
 
11
13
 
12
14
  @tags("tabular_data", "numerical_data", "visualization")
13
15
  @tasks("classification")
14
- def BivariateScatterPlots(dataset):
16
+ def BivariateScatterPlots(dataset) -> Tuple[go.Figure, RawData]:
15
17
  """
16
18
  Generates bivariate scatterplots to visually inspect relationships between pairs of numerical predictor variables
17
19
  in machine learning classification tasks.
@@ -2,6 +2,9 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+
6
+ from typing import Tuple
7
+
5
8
  import pandas as pd
6
9
  from statsmodels.stats.diagnostic import acorr_ljungbox
7
10
 
@@ -10,7 +13,7 @@ from validmind import RawData, tags, tasks
10
13
 
11
14
  @tasks("regression")
12
15
  @tags("time_series_data", "forecasting", "statistical_test", "statsmodels")
13
- def BoxPierce(dataset):
16
+ def BoxPierce(dataset) -> Tuple[pd.DataFrame, RawData]:
14
17
  """
15
18
  Detects autocorrelation in time-series data through the Box-Pierce test to validate model performance.
16
19
 
@@ -12,7 +12,7 @@ from validmind.errors import SkipTestError
12
12
 
13
13
  @tags("tabular_data", "categorical_data", "statistical_test")
14
14
  @tasks("classification")
15
- def ChiSquaredFeaturesTable(dataset, p_threshold=0.05):
15
+ def ChiSquaredFeaturesTable(dataset, p_threshold=0.05) -> pd.DataFrame:
16
16
  """
17
17
  Assesses the statistical association between categorical features and a target variable using the Chi-Squared test.
18
18
 
@@ -20,7 +20,7 @@ from validmind.vm_models import VMDataset
20
20
  @tasks("classification")
21
21
  def ClassImbalance(
22
22
  dataset: VMDataset, min_percent_threshold: int = 10
23
- ) -> Tuple[Dict[str, Any], go.Figure, bool]:
23
+ ) -> Tuple[Dict[str, Any], go.Figure, bool, RawData]:
24
24
  """
25
25
  Evaluates and quantifies class distribution imbalance in a dataset used by a machine learning model.
26
26
 
@@ -4,6 +4,7 @@
4
4
 
5
5
  import re
6
6
  from collections import Counter
7
+ from typing import Any, Dict, List, Tuple
7
8
 
8
9
  import numpy as np
9
10
 
@@ -142,7 +143,9 @@ def describe_column(df, column):
142
143
 
143
144
  @tags("tabular_data", "time_series_data", "text_data")
144
145
  @tasks("classification", "regression", "text_classification", "text_summarization")
145
- def DatasetDescription(dataset: VMDataset):
146
+ def DatasetDescription(
147
+ dataset: VMDataset,
148
+ ) -> Tuple[Dict[str, List[Dict[str, Any]]], RawData]:
146
149
  """
147
150
  Provides comprehensive analysis and statistical summaries of each column in a machine learning model's dataset.
148
151
 
@@ -2,7 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
- from typing import List
5
+
6
+ from typing import Any, Dict, List, Tuple
6
7
 
7
8
  from validmind import RawData, tags, tasks
8
9
  from validmind.vm_models import VMDataset
@@ -17,7 +18,7 @@ DATASET_LABELS = {
17
18
 
18
19
  @tags("tabular_data", "time_series_data", "text_data")
19
20
  @tasks("classification", "regression", "text_classification", "text_summarization")
20
- def DatasetSplit(datasets: List[VMDataset]):
21
+ def DatasetSplit(datasets: List[VMDataset]) -> Tuple[List[Dict[str, Any]], RawData]:
21
22
  """
22
23
  Evaluates and visualizes the distribution proportions among training, testing, and validation datasets of an ML
23
24
  model.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Any, Dict
6
+
5
7
  import pandas as pd
6
8
 
7
9
  from validmind import tags, tasks
@@ -46,7 +48,7 @@ def get_summary_statistics_categorical(df, categorical_fields):
46
48
 
47
49
  @tags("tabular_data", "time_series_data", "data_quality")
48
50
  @tasks("classification", "regression")
49
- def DescriptiveStatistics(dataset: VMDataset):
51
+ def DescriptiveStatistics(dataset: VMDataset) -> Dict[str, Any]:
50
52
  """
51
53
  Performs a detailed descriptive statistical analysis of both numerical and categorical data within a model's
52
54
  dataset.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Any, Dict, Tuple
6
+
5
7
  import pandas as pd
6
8
  from arch.unitroot import DFGLS
7
9
  from numpy.linalg import LinAlgError
@@ -16,7 +18,7 @@ logger = get_logger(__name__)
16
18
 
17
19
  @tags("time_series_data", "forecasting", "unit_root_test")
18
20
  @tasks("regression")
19
- def DickeyFullerGLS(dataset: VMDataset):
21
+ def DickeyFullerGLS(dataset: VMDataset) -> Tuple[Dict[str, Any], RawData]:
20
22
  """
21
23
  Assesses stationarity in time series data using the Dickey-Fuller GLS test to determine the order of integration.
22
24
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Any, Dict, Tuple
6
+
5
7
  import pandas as pd
6
8
 
7
9
  from validmind import tags, tasks
@@ -9,7 +11,7 @@ from validmind import tags, tasks
9
11
 
10
12
  @tags("tabular_data", "data_quality", "text_data")
11
13
  @tasks("classification", "regression")
12
- def Duplicates(dataset, min_threshold=1):
14
+ def Duplicates(dataset, min_threshold=1) -> Tuple[Dict[str, Any], bool]:
13
15
  """
14
16
  Tests dataset for duplicate entries, ensuring model reliability via data quality verification.
15
17
 
@@ -2,6 +2,9 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+
6
+ from typing import Dict
7
+
5
8
  import pandas as pd
6
9
  from statsmodels.tsa.stattools import coint
7
10
 
@@ -12,7 +15,9 @@ from validmind.vm_models import VMDataset
12
15
 
13
16
  @tags("time_series_data", "statistical_test", "forecasting")
14
17
  @tasks("regression")
15
- def EngleGrangerCoint(dataset: VMDataset, threshold: float = 0.05):
18
+ def EngleGrangerCoint(
19
+ dataset: VMDataset, threshold: float = 0.05
20
+ ) -> Dict[str, pd.DataFrame]:
16
21
  """
17
22
  Assesses the degree of co-movement between pairs of time series data using the Engle-Granger cointegration test.
18
23
 
@@ -3,6 +3,8 @@
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
5
 
6
+ from typing import Tuple
7
+
6
8
  import numpy as np
7
9
  import plotly.graph_objects as go
8
10
 
@@ -11,7 +13,7 @@ from validmind import RawData, tags, tasks
11
13
 
12
14
  @tags("tabular_data", "visualization", "correlation")
13
15
  @tasks("classification", "regression")
14
- def FeatureTargetCorrelationPlot(dataset, fig_height=600):
16
+ def FeatureTargetCorrelationPlot(dataset, fig_height=600) -> Tuple[go.Figure, RawData]:
15
17
  """
16
18
  Visualizes the correlation between input features and the model's target output in a color-coded horizontal bar
17
19
  plot.
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Any, Dict, List, Tuple
6
+
5
7
  from validmind import RawData, tags, tasks
6
8
  from validmind.vm_models import VMDataset
7
9
 
@@ -13,7 +15,7 @@ def HighCardinality(
13
15
  num_threshold: int = 100,
14
16
  percent_threshold: float = 0.1,
15
17
  threshold_type: str = "percent",
16
- ):
18
+ ) -> Tuple[List[Dict[str, Any]], bool, RawData]:
17
19
  """
18
20
  Assesses the number of unique values in categorical columns to detect high cardinality and potential overfitting.
19
21
 
@@ -2,6 +2,9 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+
6
+ from typing import Any, Dict, List, Tuple
7
+
5
8
  from validmind import RawData, tags, tasks
6
9
  from validmind.vm_models import VMDataset
7
10
 
@@ -13,7 +16,7 @@ def HighPearsonCorrelation(
13
16
  max_threshold: float = 0.3,
14
17
  top_n_correlations: int = 10,
15
18
  feature_columns: list = None,
16
- ):
19
+ ) -> Tuple[List[Dict[str, Any]], bool, RawData]:
17
20
  """
18
21
  Identifies highly correlated feature pairs in a dataset suggesting feature redundancy or multicollinearity.
19
22
 
@@ -2,6 +2,9 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+
6
+ from typing import Tuple
7
+
5
8
  import plotly.graph_objects as go
6
9
 
7
10
  from validmind import RawData, tags, tasks
@@ -22,7 +25,7 @@ def compute_outliers(series, threshold):
22
25
  @tasks("classification", "regression")
23
26
  def IQROutliersBarPlot(
24
27
  dataset: VMDataset, threshold: float = 1.5, fig_width: int = 800
25
- ):
28
+ ) -> Tuple[go.Figure, RawData]:
26
29
  """
27
30
  Visualizes outlier distribution across percentiles in numerical data using the Interquartile Range (IQR) method.
28
31
 
@@ -2,6 +2,9 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+
6
+ from typing import Any, Dict, Tuple
7
+
5
8
  from validmind import RawData, tags, tasks
6
9
  from validmind.vm_models import VMDataset
7
10
 
@@ -18,7 +21,9 @@ def compute_outliers(series, threshold=1.5):
18
21
 
19
22
  @tags("tabular_data", "numerical_data")
20
23
  @tasks("classification", "regression")
21
- def IQROutliersTable(dataset: VMDataset, threshold: float = 1.5):
24
+ def IQROutliersTable(
25
+ dataset: VMDataset, threshold: float = 1.5
26
+ ) -> Tuple[Dict[str, Any], RawData]:
22
27
  """
23
28
  Determines and summarizes outliers in numerical features using the Interquartile Range method.
24
29
 
@@ -3,7 +3,9 @@
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
5
  import itertools
6
+ from typing import Tuple
6
7
 
8
+ import matplotlib.figure
7
9
  import matplotlib.pyplot as plt
8
10
  import seaborn as sns
9
11
  from sklearn.ensemble import IsolationForest
@@ -19,7 +21,7 @@ def IsolationForestOutliers(
19
21
  random_state: int = 0,
20
22
  contamination: float = 0.1,
21
23
  feature_columns: list = None,
22
- ):
24
+ ) -> Tuple[matplotlib.figure.Figure, RawData]:
23
25
  """
24
26
  Detects outliers in a dataset using the Isolation Forest algorithm and visualizes results through scatter plots.
25
27
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Tuple
6
+
5
7
  import pandas as pd
6
8
  from statsmodels.stats.stattools import jarque_bera
7
9
 
@@ -10,7 +12,7 @@ from validmind import RawData, tags, tasks
10
12
 
11
13
  @tasks("classification", "regression")
12
14
  @tags("tabular_data", "data_distribution", "statistical_test", "statsmodels")
13
- def JarqueBera(dataset):
15
+ def JarqueBera(dataset) -> Tuple[pd.DataFrame, RawData]:
14
16
  """
15
17
  Assesses normality of dataset features in an ML model using the Jarque-Bera test.
16
18
 
@@ -2,6 +2,8 @@
2
2
  # See the LICENSE file in the root of this repository for details.
3
3
  # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
4
4
 
5
+ from typing import Any, Dict
6
+
5
7
  import pandas as pd
6
8
  from statsmodels.tsa.stattools import kpss
7
9
 
@@ -15,7 +17,7 @@ logger = get_logger(__name__)
15
17
 
16
18
  @tags("time_series_data", "stationarity", "unit_root_test", "statsmodels")
17
19
  @tasks("data_validation")
18
- def KPSS(dataset: VMDataset):
20
+ def KPSS(dataset: VMDataset) -> Dict[str, Any]:
19
21
  """
20
22
  Assesses the stationarity of time-series data in a machine learning model using the KPSS unit root test.
21
23