validmind 2.3.1__tar.gz → 2.3.5__tar.gz

{validmind-2.3.1 → validmind-2.3.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.3.1
+Version: 2.3.5
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
@@ -12,58 +12,50 @@ Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: all
+Provides-Extra: huggingface
 Provides-Extra: llm
 Provides-Extra: pytorch
 Provides-Extra: r-support
-Provides-Extra: transformers
-Requires-Dist: aiohttp[speedups] (>=3.8.4,<4.0.0)
-Requires-Dist: arch (>=5.4.0,<6.0.0)
-Requires-Dist: bert-score (>=0.3.13,<0.4.0)
-Requires-Dist: catboost (>=1.2,<2.0)
-Requires-Dist: click (>=8.0.4,<9.0.0)
-Requires-Dist: evaluate (>=0.4.0,<0.5.0)
-Requires-Dist: ipywidgets (>=8.0.6,<9.0.0)
-Requires-Dist: kaleido (>=0.2.1,<0.3.0,!=0.2.1.post1)
-Requires-Dist: langdetect (>=1.0.9,<2.0.0)
-Requires-Dist: latex2mathml (>=3.77.0,<4.0.0)
-Requires-Dist: levenshtein (>=0.21.1,<0.22.0) ; extra == "all" or extra == "llm"
-Requires-Dist: llvmlite (>=0.42.0) ; python_version >= "3.12"
+Requires-Dist: aiohttp[speedups]
+Requires-Dist: arch
+Requires-Dist: bert-score (>=0.3.13)
+Requires-Dist: catboost
+Requires-Dist: evaluate
+Requires-Dist: ipywidgets
+Requires-Dist: kaleido (>=0.2.1,!=0.2.1.post1)
+Requires-Dist: langchain-openai (>=0.1.8) ; extra == "all" or extra == "llm"
+Requires-Dist: langdetect
+Requires-Dist: latex2mathml (>=3.77.0)
 Requires-Dist: llvmlite ; python_version >= "3.8" and python_full_version <= "3.11.0"
-Requires-Dist: matplotlib (<3.8.3)
+Requires-Dist: matplotlib
 Requires-Dist: mistune (>=3.0.2,<4.0.0)
 Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numba (<0.59.0)
-Requires-Dist: numpy (>=1.23.3,<2.0.0) ; python_version >= "3.8" and python_full_version <= "3.11.0"
-Requires-Dist: numpy (>=1.26.0) ; python_version >= "3.12"
-Requires-Dist: openai (>=1.3.7,<2.0.0) ; extra == "all" or extra == "llm"
+Requires-Dist: numpy
+Requires-Dist: openai (>=1) ; extra == "all" or extra == "llm"
 Requires-Dist: pandas (>=1.1,<2)
-Requires-Dist: plotly (>=5.14.1,<6.0.0)
-Requires-Dist: plotly-express (>=0.4.1,<0.5.0)
-Requires-Dist: polars (>=0.20.15,<0.21.0)
+Requires-Dist: plotly
+Requires-Dist: plotly-express
+Requires-Dist: polars
 Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
-Requires-Dist: pypmml (>=0.9.17,<0.10.0)
-Requires-Dist: python-dotenv (>=0.20.0,<0.21.0)
-Requires-Dist: ragas (>=0.1.7,<0.2.0)
-Requires-Dist: rouge (>=1.0.1,<2.0.0)
+Requires-Dist: python-dotenv
+Requires-Dist: ragas (>=0.1.7) ; extra == "all" or extra == "llm"
+Requires-Dist: rouge (>=1)
 Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
-Requires-Dist: scikit-learn (>=1.0.2,<2.0.0)
-Requires-Dist: scipy (>=1.12) ; python_version >= "3.12"
-Requires-Dist: scipy ; python_version >= "3.8" and python_full_version <= "3.11.0"
+Requires-Dist: scikit-learn
+Requires-Dist: scipy
 Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
-Requires-Dist: seaborn (>=0.11.2,<0.12.0)
-Requires-Dist: selfcheckgpt (>=0.1.7,<0.2.0)
+Requires-Dist: seaborn
 Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: shap (>=0.42.0,<0.43.0)
-Requires-Dist: statsmodels (>=0.13.5,<0.14.0)
+Requires-Dist: statsmodels
 Requires-Dist: tabulate (>=0.8.9,<0.9.0)
 Requires-Dist: textblob (>=0.18.0.post0,<0.19.0)
-Requires-Dist: textstat (>=0.7.3,<0.8.0)
 Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "pytorch"
-Requires-Dist: torchmetrics (>=1.1.1,<2.0.0) ; extra == "all" or extra == "llm"
-Requires-Dist: tqdm (>=4.64.0,<5.0.0)
-Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "llm" or extra == "transformers"
+Requires-Dist: tqdm
+Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "llm" or extra == "huggingface"
 Requires-Dist: xgboost (>=1.5.2,<3)
-Requires-Dist: ydata-profiling (>=4.7.0,<5.0.0)
+Requires-Dist: ydata-profiling
 Description-Content-Type: text/markdown
 
 # ValidMind Developer Framework
@@ -79,3 +71,45 @@ descriptions of your dataset to testing your models for weak spots and overfit a
 Framework helps you automate the generation of model documentation by feeding the ValidMind platform with
 documentation artifacts and test results to the ValidMind platform.
 
+## Installation
+
+To install the ValidMind Developer Framework and all optional dependencies, run:
+
+```bash
+pip install validmind[all]
+```
+
+To install the Developer Framework without optional dependencies (core functionality only), run:
+
+```bash
+pip install validmind
+```
+
+### Extra dependencies
+
+The Developer Framework has optional dependencies that can be installed separately to support additional model types and tests.
+
+- **LLM Support**: To be able to run tests for Large Language Models (LLMs), install the `llm` extra:
+
+    ```bash
+    pip install validmind[llm]
+    ```
+
+- **PyTorch Models**: To use pytorch models with the Developer Framework, install the `torch` extra:
+
+    ```bash
+    pip install validmind[torch]
+    ```
+
+- **Hugging Face Transformers**: To use Hugging Face Transformers models with the Developer Framework, install the `transformers` extra:
+
+    ```bash
+    pip install validmind[transformers]
+    ```
+
+- **R Models**: To use R models with the Developer Framework, install the `r` extra:
+
+    ```bash
+    pip install validmind[r-support]
+    ```
+

validmind-2.3.5/README.pypi.md

@@ -0,0 +1,54 @@
+# ValidMind Developer Framework
+
+ValidMind’s Python Developer Framework is a library of developer tools and methods designed to automate
+the documentation and validation of your models.
+
+The Developer Framework is designed to be model agnostic. If your model is built in Python, ValidMind's
+Python library will provide all the standard functionality without requiring your developers to rewrite any functions.
+
+The Developer Framework provides a rich suite of documentation tools and test suites, from documenting
+descriptions of your dataset to testing your models for weak spots and overfit areas. The Developer
+Framework helps you automate the generation of model documentation by feeding the ValidMind platform with
+documentation artifacts and test results to the ValidMind platform.
+
+## Installation
+
+To install the ValidMind Developer Framework and all optional dependencies, run:
+
+```bash
+pip install validmind[all]
+```
+
+To install the Developer Framework without optional dependencies (core functionality only), run:
+
+```bash
+pip install validmind
+```
+
+### Extra dependencies
+
+The Developer Framework has optional dependencies that can be installed separately to support additional model types and tests.
+
+- **LLM Support**: To be able to run tests for Large Language Models (LLMs), install the `llm` extra:
+
+    ```bash
+    pip install validmind[llm]
+    ```
+
+- **PyTorch Models**: To use pytorch models with the Developer Framework, install the `torch` extra:
+
+    ```bash
+    pip install validmind[torch]
+    ```
+
+- **Hugging Face Transformers**: To use Hugging Face Transformers models with the Developer Framework, install the `transformers` extra:
+
+    ```bash
+    pip install validmind[transformers]
+    ```
+
+- **R Models**: To use R models with the Developer Framework, install the `r` extra:
+
+    ```bash
+    pip install validmind[r-support]
+    ```

{validmind-2.3.1 → validmind-2.3.5}/pyproject.toml

@@ -10,71 +10,56 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.3.1"
+version = "2.3.5"
 
 [tool.poetry.dependencies]
 python = ">=3.8.1,<3.12"
-aiohttp = {extras = ["speedups"], version = "^3.8.4"}
-arch = "^5.4.0"
-bert-score = "^0.3.13"
-catboost = "^1.2"
-click = "^8.0.4"
-evaluate = "^0.4.0"
-ipywidgets = "^8.0.6"
-kaleido = "^0.2.1,!=0.2.1.post1"
-langdetect = "^1.0.9"
-latex2mathml = "^3.77.0"
-levenshtein = {version = "^0.21.1", optional = true}
-llvmlite = [
-  {version = "*", python = ">=3.8,<=3.11"},
-  {version = ">=0.42.0", python = ">=3.12"},
-]
-matplotlib = "<3.8.3"
+aiohttp = {extras = ["speedups"], version = "*"}
+arch = "*"
+bert-score = ">=0.3.13"
+catboost = "*"
+evaluate = "*"
+ipywidgets = "*"
+kaleido = ">=0.2.1,!=0.2.1.post1"
+langchain-openai = { version = ">=0.1.8", optional = true }
+langdetect = "*"
+latex2mathml = ">=3.77.0"
+llvmlite = {version = "*", python = ">=3.8,<=3.11"}
+matplotlib = "*"
 mistune = "^3.0.2"
 nltk = "^3.8.1"
 numba = "<0.59.0" # TODO: https://github.com/validmind/developer-framework/pull/28
-numpy = [
-  {version = "^1.23.3", python = ">=3.8,<=3.11"},
-  {version = ">=1.26.0", python = ">=3.12"},
-]
-openai = {version = "^1.3.7", optional = true}
+numpy = "*"
+openai = {version = ">=1", optional = true}
 pandas = ">=1.1,<2" # TODO small issues blocking 2.0 but it would speed things up to use arrow backend
-plotly = "^5.14.1"
-plotly-express = "^0.4.1"
-polars = "^0.20.15"
+plotly = "*"
+plotly-express = "*"
+polars = "*"
 pycocoevalcap = {version = "^1.2", optional = true}
-pypmml = "^0.9.17"
-python-dotenv = "^0.20.0"
-ragas = "^0.1.7"
-rouge = "^1.0.1"
+python-dotenv = "*"
+ragas = {version = ">=0.1.7", optional = true}
+rouge = ">=1"
 rpy2 = {version = "^3.5.10", optional = true}
-scikit-learn = "^1.0.2"
-scipy = [
-  {version = "*", python = ">=3.8,<=3.11"},
-  {version = ">=1.12", python = ">=3.12"},
-]
+scikit-learn = "*"
+scipy = "*"
 scorecardpy = "^0.1.9.6"
-seaborn = "^0.11.2"
-selfcheckgpt = "^0.1.7"
+seaborn = "*"
 sentry-sdk = "^1.24.0"
 shap = "^0.42.0"
-statsmodels = "^0.13.5"
+statsmodels = "*"
 tabulate = "^0.8.9"
 textblob = "^0.18.0.post0"
-textstat = "^0.7.3"
 torch = {version = ">=1.10.0", optional = true}
-torchmetrics = {version = "^1.1.1", optional = true}
-tqdm = "^4.64.0"
+tqdm = "*"
 transformers = {version = "^4.32.0", optional = true}
 xgboost = ">=1.5.2,<3"
-ydata-profiling = "^4.7.0"
+ydata-profiling = "*"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.1.0"
-click = "^8.1.3"
+click = "*"
 cython = "^0.29.34"
 flake8 = "^4.0.1"
-gradio = "^3.43.2"
 ipykernel = "^6.22.0"
 isort = "^5.12.0"
 jupyter = "^1.0.0"
@@ -82,7 +67,6 @@ openai = "^1.3.7"
 papermill = "^2.4.0"
 pdoc = "^14.4.0"
 pre-commit = "^3.3.3"
-pytest = "^5.2"
 sphinx = "^6.1.3"
 sphinx-markdown-builder = "^0.5.5"
 sphinx-rtd-theme = "^1.2.0"
@@ -95,20 +79,20 @@ all = [
   "torch",
   "transformers",
   "pycocoevalcap",
-  "torchmetrics",
-  "levenshtein",
+  "ragas",
+  "langchain-openai",
 ]
 llm = [
   "torch",
   "transformers",
   "openai",
   "pycocoevalcap",
-  "torchmetrics",
-  "levenshtein",
+  "ragas",
+  "langchain-openai",
 ]
 pytorch = ["torch"]
 r-support = ["rpy2"]
-transformers = ["transformers"]
+huggingface = ["transformers"]
 
 [build-system]
 build-backend = "poetry.core.masonry.api"

{validmind-2.3.1 → validmind-2.3.5}/validmind/__init__.py

@@ -60,7 +60,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
-from .tests.decorator import metric, tags, tasks
+from .tests.decorator import metric, tags, tasks, test
 from .utils import run_async  # noqa: E402
 
 
@@ -112,6 +112,7 @@ __all__ = [  # noqa
     "run_test_suite",
     "tags",
     "tasks",
+    "test",
     "tests",
     "test_suites",
     "vm_models",

validmind-2.3.5/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.3.5"

validmind-2.3.5/validmind/datasets/regression/fred_timeseries.py

@@ -0,0 +1,272 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+
+import pandas as pd
+
+current_path = os.path.dirname(os.path.abspath(__file__))
+mortgage30us_path = os.path.join(current_path, "datasets", "fred", "MORTGAGE30US.csv")
+fedfunds_path = os.path.join(current_path, "datasets", "fred", "FEDFUNDS.csv")
+gs10_path = os.path.join(current_path, "datasets", "fred", "GS10.csv")
+unrate_path = os.path.join(current_path, "datasets", "fred", "UNRATE.csv")
+
+target_column = "MORTGAGE30US"
+feature_columns = ["FEDFUNDS", "GS10", "UNRATE"]
+
+
+def get_common_date_range(dfs):
+    start_dates = [df.index.min() for df in dfs]
+    end_dates = [df.index.max() for df in dfs]
+
+    common_start_date = max(start_dates)
+    common_end_date = min(end_dates)
+
+    return common_start_date, common_end_date
+
+
+def align_date_range(dfs, start_date, end_date):
+    return [df.loc[start_date:end_date] for df in dfs]
+
+
+def load_data():
+    mortgage30us = pd.read_csv(
+        mortgage30us_path, parse_dates=["DATE"], index_col="DATE"
+    )
+    fedfunds = pd.read_csv(fedfunds_path, parse_dates=["DATE"], index_col="DATE")
+    gs10 = pd.read_csv(gs10_path, parse_dates=["DATE"], index_col="DATE")
+    unrate = pd.read_csv(unrate_path, parse_dates=["DATE"], index_col="DATE")
+
+    # Resample mortgage30us to monthly frequency
+    mortgage30us = mortgage30us.resample("MS").last()
+
+    # Get the common date range
+    common_start_date, common_end_date = get_common_date_range(
+        [mortgage30us, fedfunds, gs10, unrate]
+    )
+
+    # Align the date range for all dataframes
+    mortgage30us, fedfunds, gs10, unrate = align_date_range(
+        [mortgage30us, fedfunds, gs10, unrate], common_start_date, common_end_date
+    )
+
+    # Combine into a single DataFrame
+    df = pd.concat([mortgage30us, fedfunds, gs10, unrate], axis=1, join="inner")
+    df.columns = [target_column] + feature_columns
+
+    return df
+
+
+# Convert data back to levels
+def convert_to_levels(diff_df, original_df, target_column):
+    """
+    Convert differenced data back to original levels.
+    """
+    previous_values = original_df[target_column].shift(1).dropna()
+    levels_df = diff_df.add(previous_values, axis=0)
+    return levels_df
+
+
+def get_demo_test_config(test_suite=None):
+
+    default_config = {}
+
+    default_config["validmind.data_validation.TimeSeriesDescription"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesLinePlot"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesMissingValues"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config["validmind.data_validation.SeasonalDecompose"] = {
+        "inputs": {
+            "dataset": "raw_ds",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TimeSeriesDescriptiveStatistics:train_diff_data"
+    ] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        }
+    }
+    default_config[
+        "validmind.data_validation.TimeSeriesDescriptiveStatistics:test_diff_data"
+    ] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        }
+    }
+    default_config["validmind.data_validation.TimeSeriesOutliers:train_diff_data"] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        },
+        "params": {"zscore_threshold": 4},
+    }
+    default_config["validmind.data_validation.TimeSeriesOutliers:test_diff_data"] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        },
+        "params": {"zscore_threshold": 4},
+    }
+    default_config["validmind.data_validation.TimeSeriesHistogram:train_diff_data"] = {
+        "inputs": {
+            "dataset": "train_diff_ds",
+        },
+        "params": {"nbins": 100},
+    }
+    default_config["validmind.data_validation.TimeSeriesHistogram:test_diff_data"] = {
+        "inputs": {
+            "dataset": "test_diff_ds",
+        },
+        "params": {"nbins": 100},
+    }
+    default_config["validmind.data_validation.DatasetSplit"] = {
+        "inputs": {
+            "datasets": ["train_diff_ds", "test_diff_ds"],
+        }
+    }
+    default_config["validmind.model_validation.ModelMetadataComparison"] = {
+        "inputs": {
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionErrorsComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionErrorsComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionR2SquareComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.RegressionR2SquareComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesR2SquareBySegments:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesR2SquareBySegments:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        },
+        "params": {
+            "segments": {
+                "start_date": ["2012-11-01", "2018-02-01"],
+                "end_date": ["2018-01-01", "2023-03-01"],
+            }
+        },
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionsPlot:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config["validmind.model_validation.TimeSeriesPredictionsPlot:test_data"] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionWithCI:random_forests_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "random_forests_model",
+        }
+    }
+    default_config[
+        "validmind.model_validation.TimeSeriesPredictionWithCI:gradient_boosting_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "gradient_boosting_model",
+        }
+    }
+    default_config["validmind.model_validation.ModelPredictionResiduals:train_data"] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config["validmind.model_validation.ModelPredictionResiduals:test_data"] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.FeatureImportanceComparison:train_data"
+    ] = {
+        "inputs": {
+            "datasets": ["train_ds", "train_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.FeatureImportanceComparison:test_data"
+    ] = {
+        "inputs": {
+            "datasets": ["test_ds", "test_ds"],
+            "models": ["random_forests_model", "gradient_boosting_model"],
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.PermutationFeatureImportance:random_forests_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "random_forests_model",
+        }
+    }
+    default_config[
+        "validmind.model_validation.sklearn.PermutationFeatureImportance:gradient_boosting_model"
+    ] = {
+        "inputs": {
+            "dataset": "test_ds",
+            "model": "gradient_boosting_model",
+        }
+    }
+
+    return default_config

{validmind-2.3.1 → validmind-2.3.5}/validmind/test_suites/__init__.py

@@ -182,7 +182,6 @@ def describe_suite(test_suite_id: str, verbose=False):
                     "Test Suite Section": "",
                     "Test ID": item,
                     "Test Name": test.__name__,
-                    "Test Type": test.test_type,
                 }
             )
         elif isinstance(item, dict):
@@ -195,7 +194,6 @@ def describe_suite(test_suite_id: str, verbose=False):
                         "Test Suite Section": item["section_id"],
                         "Test ID": test_id,
                         "Test Name": test_id_to_name(test_id),
-                        "Test Type": test.test_type,
                     }
                 )
         else: