validmind 2.3.1__tar.gz → 2.3.3__tar.gz

{validmind-2.3.1 → validmind-2.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.3.1
+Version: 2.3.3
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez

{validmind-2.3.1 → validmind-2.3.3}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.3.1"
+version = "2.3.3"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.12"

{validmind-2.3.1 → validmind-2.3.3}/validmind/__init__.py

@@ -60,7 +60,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .tests.decorator import metric, tags, tasks
+from .tests.decorator import metric, tags, tasks, test
 from .utils import run_async  # noqa: E402
 
 
@@ -112,6 +112,7 @@ __all__ = [  # noqa
     "run_test_suite",
     "tags",
     "tasks",
+    "test",
     "tests",
     "test_suites",
     "vm_models",

validmind-2.3.3/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.3.3"

{validmind-2.3.1 → validmind-2.3.3}/validmind/test_suites/__init__.py

@@ -182,7 +182,6 @@ def describe_suite(test_suite_id: str, verbose=False):
                     "Test Suite Section": "",
                     "Test ID": item,
                     "Test Name": test.__name__,
-                    "Test Type": test.test_type,
                 }
             )
         elif isinstance(item, dict):
@@ -195,7 +194,6 @@ def describe_suite(test_suite_id: str, verbose=False):
                         "Test Suite Section": item["section_id"],
                         "Test ID": test_id,
                         "Test Name": test_id_to_name(test_id),
-                        "Test Type": test.test_type,
                     }
                 )
         else:

{validmind-2.3.1 → validmind-2.3.3}/validmind/tests/__init__.py

@@ -30,7 +30,9 @@ from ..utils import (
     test_id_to_name,
 )
 from ..vm_models import TestContext, TestInput
-from .decorator import metric, tags, tasks
+from .__types__ import TestID
+from .decorator import tags, tasks
+from .decorator import test as test_decorator
 from .test_providers import LocalTestProvider, TestProvider
 
 logger = get_logger(__name__)
@@ -84,7 +86,6 @@ def _pretty_list_tests(tests, truncate=True):
         {
             "ID": test_id,
             "Name": test_id_to_name(test_id),
-            "Test Type": __test_classes[test_id].test_type,
             "Description": _test_description(__test_classes[test_id], truncate),
             "Required Inputs": __test_classes[test_id].required_inputs,
             "Params": __test_classes[test_id].default_params or {},
@@ -340,7 +341,7 @@ def load_test(test_id: str, reload=False):
         # if its a function, we decorate it and then load the class
         # TODO: simplify this as we move towards all functional metrics
         # "_" is used here so it doesn't conflict with other test ids
-        metric("_")(test)
+        test_decorator("_")(test)
         test = __custom_tests["_"]
 
     test.test_id = f"{test_id}:{result_id}" if result_id else test_id
@@ -348,7 +349,7 @@ def load_test(test_id: str, reload=False):
     return test
 
 
-def describe_test(test_id: str = None, raw: bool = False, show: bool = True):
+def describe_test(test_id: TestID = None, raw: bool = False, show: bool = True):
     """Get or show details about the test
 
     This function can be used to see test details including the test name, description,
@@ -365,7 +366,6 @@ def describe_test(test_id: str = None, raw: bool = False, show: bool = True):
     details = {
         "ID": test_id,
         "Name": test_id_to_name(test_id),
-        "Test Type": test.test_type,
         "Required Inputs": test.required_inputs,
         "Params": test.default_params or {},
         "Description": inspect.getdoc(test).strip() or "",
@@ -407,7 +407,7 @@ def describe_test(test_id: str = None, raw: bool = False, show: bool = True):
 
 
 def run_test(
-    test_id: str = None,
+    test_id: TestID = None,
     name: str = None,
     unit_metrics: list = None,
     params: dict = None,
@@ -451,7 +451,7 @@ def run_test(
 
     if unit_metrics:
         metric_id_name = "".join(word[0].upper() + word[1:] for word in name.split())
-        test_id = f"validmind.composite_metric.{metric_id_name}"
+        test_id = f"validmind.composite_test.{metric_id_name}"
 
         error, TestClass = load_composite_metric(
             unit_metrics=unit_metrics, metric_name=metric_id_name

validmind-2.3.3/validmind/tests/__types__.py

@@ -0,0 +1,170 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Literal types for test IDs.
+
+This module is auto-generated by running `make generate-test-id-types`.
+Should not be modified manually.
+"""
+
+from typing import Literal
+
+TestID = Literal[
+    "validmind.prompt_validation.Bias",
+    "validmind.prompt_validation.Clarity",
+    "validmind.prompt_validation.Specificity",
+    "validmind.prompt_validation.Robustness",
+    "validmind.prompt_validation.NegativeInstruction",
+    "validmind.prompt_validation.Conciseness",
+    "validmind.prompt_validation.Delimitation",
+    "validmind.model_validation.BertScore",
+    "validmind.model_validation.RegardScore",
+    "validmind.model_validation.BleuScore",
+    "validmind.model_validation.RegressionResidualsPlot",
+    "validmind.model_validation.FeaturesAUC",
+    "validmind.model_validation.ContextualRecall",
+    "validmind.model_validation.MeteorScore",
+    "validmind.model_validation.RougeScore",
+    "validmind.model_validation.ModelMetadata",
+    "validmind.model_validation.ClusterSizeDistribution",
+    "validmind.model_validation.TokenDisparity",
+    "validmind.model_validation.ToxicityScore",
+    "validmind.model_validation.embeddings.CosineSimilarityComparison",
+    "validmind.model_validation.embeddings.EmbeddingsVisualization2D",
+    "validmind.model_validation.embeddings.StabilityAnalysisRandomNoise",
+    "validmind.model_validation.embeddings.TSNEComponentsPairwisePlots",
+    "validmind.model_validation.embeddings.CosineSimilarityDistribution",
+    "validmind.model_validation.embeddings.PCAComponentsPairwisePlots",
+    "validmind.model_validation.embeddings.CosineSimilarityHeatmap",
+    "validmind.model_validation.embeddings.StabilityAnalysisTranslation",
+    "validmind.model_validation.embeddings.EuclideanDistanceComparison",
+    "validmind.model_validation.embeddings.ClusterDistribution",
+    "validmind.model_validation.embeddings.EuclideanDistanceHeatmap",
+    "validmind.model_validation.embeddings.StabilityAnalysis",
+    "validmind.model_validation.embeddings.StabilityAnalysisKeyword",
+    "validmind.model_validation.embeddings.StabilityAnalysisSynonyms",
+    "validmind.model_validation.embeddings.DescriptiveAnalytics",
+    "validmind.model_validation.ragas.ContextEntityRecall",
+    "validmind.model_validation.ragas.Faithfulness",
+    "validmind.model_validation.ragas.AspectCritique",
+    "validmind.model_validation.ragas.AnswerSimilarity",
+    "validmind.model_validation.ragas.AnswerCorrectness",
+    "validmind.model_validation.ragas.ContextRecall",
+    "validmind.model_validation.ragas.ContextRelevancy",
+    "validmind.model_validation.ragas.ContextPrecision",
+    "validmind.model_validation.ragas.AnswerRelevance",
+    "validmind.model_validation.sklearn.RegressionModelsPerformanceComparison",
+    "validmind.model_validation.sklearn.AdjustedMutualInformation",
+    "validmind.model_validation.sklearn.SilhouettePlot",
+    "validmind.model_validation.sklearn.RobustnessDiagnosis",
+    "validmind.model_validation.sklearn.AdjustedRandIndex",
+    "validmind.model_validation.sklearn.SHAPGlobalImportance",
+    "validmind.model_validation.sklearn.ConfusionMatrix",
+    "validmind.model_validation.sklearn.HomogeneityScore",
+    "validmind.model_validation.sklearn.CompletenessScore",
+    "validmind.model_validation.sklearn.OverfitDiagnosis",
+    "validmind.model_validation.sklearn.ClusterPerformanceMetrics",
+    "validmind.model_validation.sklearn.PermutationFeatureImportance",
+    "validmind.model_validation.sklearn.FowlkesMallowsScore",
+    "validmind.model_validation.sklearn.MinimumROCAUCScore",
+    "validmind.model_validation.sklearn.ClusterCosineSimilarity",
+    "validmind.model_validation.sklearn.PrecisionRecallCurve",
+    "validmind.model_validation.sklearn.ClassifierPerformance",
+    "validmind.model_validation.sklearn.VMeasure",
+    "validmind.model_validation.sklearn.MinimumF1Score",
+    "validmind.model_validation.sklearn.ROCCurve",
+    "validmind.model_validation.sklearn.RegressionR2Square",
+    "validmind.model_validation.sklearn.RegressionErrors",
+    "validmind.model_validation.sklearn.ClusterPerformance",
+    "validmind.model_validation.sklearn.TrainingTestDegradation",
+    "validmind.model_validation.sklearn.HyperParametersTuning",
+    "validmind.model_validation.sklearn.KMeansClustersOptimization",
+    "validmind.model_validation.sklearn.ModelsPerformanceComparison",
+    "validmind.model_validation.sklearn.WeakspotsDiagnosis",
+    "validmind.model_validation.sklearn.PopulationStabilityIndex",
+    "validmind.model_validation.sklearn.MinimumAccuracy",
+    "validmind.model_validation.statsmodels.RegressionModelsCoeffs",
+    "validmind.model_validation.statsmodels.BoxPierce",
+    "validmind.model_validation.statsmodels.RegressionCoeffsPlot",
+    "validmind.model_validation.statsmodels.RegressionModelSensitivityPlot",
+    "validmind.model_validation.statsmodels.RegressionModelForecastPlotLevels",
+    "validmind.model_validation.statsmodels.ScorecardHistogram",
+    "validmind.model_validation.statsmodels.LJungBox",
+    "validmind.model_validation.statsmodels.JarqueBera",
+    "validmind.model_validation.statsmodels.KolmogorovSmirnov",
+    "validmind.model_validation.statsmodels.ShapiroWilk",
+    "validmind.model_validation.statsmodels.CumulativePredictionProbabilities",
+    "validmind.model_validation.statsmodels.RegressionFeatureSignificance",
+    "validmind.model_validation.statsmodels.RegressionModelSummary",
+    "validmind.model_validation.statsmodels.Lilliefors",
+    "validmind.model_validation.statsmodels.RunsTest",
+    "validmind.model_validation.statsmodels.RegressionPermutationFeatureImportance",
+    "validmind.model_validation.statsmodels.PredictionProbabilitiesHistogram",
+    "validmind.model_validation.statsmodels.AutoARIMA",
+    "validmind.model_validation.statsmodels.GINITable",
+    "validmind.model_validation.statsmodels.RegressionModelForecastPlot",
+    "validmind.model_validation.statsmodels.DurbinWatsonTest",
+    "validmind.data_validation.MissingValuesRisk",
+    "validmind.data_validation.IQROutliersTable",
+    "validmind.data_validation.BivariateFeaturesBarPlots",
+    "validmind.data_validation.Skewness",
+    "validmind.data_validation.Duplicates",
+    "validmind.data_validation.MissingValuesBarPlot",
+    "validmind.data_validation.DatasetDescription",
+    "validmind.data_validation.ZivotAndrewsArch",
+    "validmind.data_validation.ScatterPlot",
+    "validmind.data_validation.TimeSeriesOutliers",
+    "validmind.data_validation.TabularCategoricalBarPlots",
+    "validmind.data_validation.AutoStationarity",
+    "validmind.data_validation.DescriptiveStatistics",
+    "validmind.data_validation.ANOVAOneWayTable",
+    "validmind.data_validation.TargetRateBarPlots",
+    "validmind.data_validation.PearsonCorrelationMatrix",
+    "validmind.data_validation.FeatureTargetCorrelationPlot",
+    "validmind.data_validation.TabularNumericalHistograms",
+    "validmind.data_validation.IsolationForestOutliers",
+    "validmind.data_validation.ChiSquaredFeaturesTable",
+    "validmind.data_validation.HighCardinality",
+    "validmind.data_validation.MissingValues",
+    "validmind.data_validation.PhillipsPerronArch",
+    "validmind.data_validation.RollingStatsPlot",
+    "validmind.data_validation.TabularDescriptionTables",
+    "validmind.data_validation.AutoMA",
+    "validmind.data_validation.UniqueRows",
+    "validmind.data_validation.TooManyZeroValues",
+    "validmind.data_validation.HighPearsonCorrelation",
+    "validmind.data_validation.ACFandPACFPlot",
+    "validmind.data_validation.BivariateHistograms",
+    "validmind.data_validation.WOEBinTable",
+    "validmind.data_validation.HeatmapFeatureCorrelations",
+    "validmind.data_validation.TimeSeriesFrequency",
+    "validmind.data_validation.DatasetSplit",
+    "validmind.data_validation.SpreadPlot",
+    "validmind.data_validation.TimeSeriesLinePlot",
+    "validmind.data_validation.KPSS",
+    "validmind.data_validation.AutoSeasonality",
+    "validmind.data_validation.BivariateScatterPlots",
+    "validmind.data_validation.EngleGrangerCoint",
+    "validmind.data_validation.TimeSeriesMissingValues",
+    "validmind.data_validation.TimeSeriesHistogram",
+    "validmind.data_validation.LaggedCorrelationHeatmap",
+    "validmind.data_validation.SeasonalDecompose",
+    "validmind.data_validation.WOEBinPlots",
+    "validmind.data_validation.ClassImbalance",
+    "validmind.data_validation.IQROutliersBarPlot",
+    "validmind.data_validation.DFGLSArch",
+    "validmind.data_validation.AutoAR",
+    "validmind.data_validation.TabularDateTimeHistograms",
+    "validmind.data_validation.ADF",
+    "validmind.data_validation.nlp.Toxicity",
+    "validmind.data_validation.nlp.PolarityAndSubjectivity",
+    "validmind.data_validation.nlp.Punctuations",
+    "validmind.data_validation.nlp.Sentiment",
+    "validmind.data_validation.nlp.CommonWords",
+    "validmind.data_validation.nlp.Hashtags",
+    "validmind.data_validation.nlp.LanguageDetection",
+    "validmind.data_validation.nlp.Mentions",
+    "validmind.data_validation.nlp.TextDescription",
+    "validmind.data_validation.nlp.StopWords",
+]

{validmind-2.3.1 → validmind-2.3.3}/validmind/tests/decorator.py

@@ -226,6 +226,18 @@ def _get_save_func(func, test_id):
 
 
 def metric(func_or_id):
+    """
+    DEPRECATED, use @vm.test instead
+    """
+    # print a deprecation notice and call the test() function instead
+    logger.warning(
+        "The @vm.metric decorator is deprecated and will be removed in a future release. "
+        "Please use @vm.test instead."
+    )
+    return test(func_or_id)
+
+
+def test(func_or_id):
     """Decorator for creating and registering metrics with the ValidMind framework.
 
     Creates a metric object and registers it with ValidMind under the provided ID. If

{validmind-2.3.1 → validmind-2.3.3}/validmind/utils.py

@@ -4,6 +4,7 @@
 
 import asyncio
 import difflib
+import inspect
 import json
 import math
 import re
@@ -458,3 +459,36 @@ def md_to_html(md: str, mathml=False) -> str:
     )
 
     return html
+
+
+def inspect_obj(obj):
+    # Filtering only attributes
+    print(len("Attributes:") * "-")
+    print("Attributes:")
+    print(len("Attributes:") * "-")
+
+    # Get only attributes (not methods)
+    attributes = [
+        attr
+        for attr in dir(obj)
+        if not callable(getattr(obj, attr)) and not attr.startswith("__")
+    ]
+    for attr in attributes:
+        print(f"{attr}")
+
+    # Filtering only methods using inspect and displaying their parameters
+    print("\nMethods with Parameters:")
+
+    # Get only methods (functions) using inspect.ismethod
+    methods = inspect.getmembers(obj, predicate=inspect.ismethod)
+    print("Methods:")
+    for name, method in methods:
+        # Get the signature of the method
+        sig = inspect.signature(method)
+        print(len(f"{name}") * "-")
+        print(f"{name}")
+        print(len(f"{name}") * "-")
+        print("Parameters:")
+        # Loop through the parameters and print detailed information
+        for param_name, param in sig.parameters.items():
+            print(f"{param_name} - ({param.default})")

validmind-2.3.1/validmind/__version__.py

@@ -1 +0,0 @@
-__version__ = "2.3.1"