validmind 2.0.0__tar.gz → 2.0.7__tar.gz

{validmind-2.0.0 → validmind-2.0.7}/PKG-INFO

@@ -1,16 +1,17 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.0.0
+Version: 2.0.7
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
-Requires-Python: >=3.8,<3.11
+Requires-Python: >=3.8,<3.12
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
 Provides-Extra: all
 Provides-Extra: llm
 Provides-Extra: pytorch
@@ -21,27 +22,22 @@ Requires-Dist: arch (>=5.4.0,<6.0.0)
 Requires-Dist: bert-score (>=0.3.13,<0.4.0)
 Requires-Dist: catboost (>=1.2,<2.0)
 Requires-Dist: click (>=8.0.4,<9.0.0)
-Requires-Dist: datasets (>=2.14.5,<3.0.0) ; extra == "all" or extra == "llm"
-Requires-Dist: dython (>=0.7.1,<0.8.0)
 Requires-Dist: evaluate (>=0.4.0,<0.5.0)
-Requires-Dist: ipython (==7.34.0)
 Requires-Dist: ipywidgets (>=8.0.6,<9.0.0)
-Requires-Dist: jupyter (>=1.0.0,<2.0.0)
-Requires-Dist: kaleido (==0.2.1)
+Requires-Dist: kaleido (>=0.2.1,<0.3.0,!=0.2.1.post1)
 Requires-Dist: langdetect (>=1.0.9,<2.0.0)
 Requires-Dist: levenshtein (>=0.21.1,<0.22.0) ; extra == "all" or extra == "llm"
 Requires-Dist: markdown (>=3.4.3,<4.0.0)
-Requires-Dist: matplotlib (<3.8)
+Requires-Dist: matplotlib (<3.8.3)
 Requires-Dist: nltk (>=3.8.1,<4.0.0)
+Requires-Dist: numba (<0.59.0)
 Requires-Dist: numpy (>=1.23.3,<2.0.0)
 Requires-Dist: openai (>=1.3.7,<2.0.0) ; extra == "all" or extra == "llm"
 Requires-Dist: pandas (==1.5.3)
-Requires-Dist: pandas-profiling (>=3.6.6,<4.0.0)
-Requires-Dist: pdoc (>=13.1.1,<14.0.0)
 Requires-Dist: plotly (>=5.14.1,<6.0.0)
 Requires-Dist: plotly-express (>=0.4.1,<0.5.0)
+Requires-Dist: polars (>=0.20.15,<0.21.0)
 Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
-Requires-Dist: pydantic (>=1.9.1,<2.0.0)
 Requires-Dist: pypmml (>=0.9.17,<0.10.0)
 Requires-Dist: python-dotenv (>=0.20.0,<0.21.0)
 Requires-Dist: requests (>=2.27.1,<3.0.0)
@@ -50,7 +46,7 @@ Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
 Requires-Dist: scikit-learn (>=1.0.2,<2.0.0)
 Requires-Dist: scorecardpy (>=0.1.9.6,<0.2.0.0)
 Requires-Dist: seaborn (>=0.11.2,<0.12.0)
-Requires-Dist: sentencepiece (>=0.1.99,<0.2.0)
+Requires-Dist: selfcheckgpt (>=0.1.7,<0.2.0)
 Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: shap (>=0.42.0,<0.43.0)
 Requires-Dist: statsmodels (>=0.13.5,<0.14.0)
@@ -60,8 +56,8 @@ Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "
 Requires-Dist: torchmetrics (>=1.1.1,<2.0.0) ; extra == "all" or extra == "llm"
 Requires-Dist: tqdm (>=4.64.0,<5.0.0)
 Requires-Dist: transformers (>=4.32.0,<5.0.0) ; extra == "all" or extra == "llm" or extra == "transformers"
-Requires-Dist: wget (>=3.2,<4.0)
 Requires-Dist: xgboost (>=1.5.2,<2.0.0)
+Requires-Dist: ydata-profiling (>=4.7.0,<5.0.0)
 Description-Content-Type: text/markdown
 
 # ValidMind Developer Framework

{validmind-2.0.0 → validmind-2.0.7}/pyproject.toml

@@ -10,7 +10,7 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.0.0"
+version = "2.0.7"
 
 [tool.poetry.dependencies]
 aiohttp = {extras = ["speedups"], version = "^3.8.4"}
@@ -18,29 +18,24 @@ arch = "^5.4.0"
 bert-score = "^0.3.13"
 catboost = "^1.2"
 click = "^8.0.4"
-datasets = {version = "^2.14.5", optional = true}
-dython = "^0.7.1"
 evaluate = "^0.4.0"
-ipython = "7.34.0"
 ipywidgets = "^8.0.6"
-jupyter = "^1.0.0"
-kaleido = "0.2.1"
+kaleido = "^0.2.1,!=0.2.1.post1"
 langdetect = "^1.0.9"
 levenshtein = {version = "^0.21.1", optional = true}
 markdown = "^3.4.3"
-matplotlib = "<3.8"
+matplotlib = "<3.8.3"
 nltk = "^3.8.1"
+numba = "<0.59.0" # TODO: https://github.com/validmind/developer-framework/pull/28
 numpy = "^1.23.3"
 openai = {version = "^1.3.7", optional = true}
 pandas = "1.5.3"
-pandas-profiling = "^3.6.6"
-pdoc = "^13.1.1"
 plotly = "^5.14.1"
 plotly-express = "^0.4.1"
+polars = "^0.20.15"
 pycocoevalcap = {version = "^1.2", optional = true}
-pydantic = "^1.9.1"
 pypmml = "^0.9.17"
-python = ">=3.8,<3.11"
+python = ">=3.8,<3.12"
 python-dotenv = "^0.20.0"
 requests = "^2.27.1"
 rouge = "^1.0.1"
@@ -48,7 +43,7 @@ rpy2 = {version = "^3.5.10", optional = true}
 scikit-learn = "^1.0.2"
 scorecardpy = "^0.1.9.6"
 seaborn = "^0.11.2"
-sentencepiece = "^0.1.99"
+selfcheckgpt = "^0.1.7"
 sentry-sdk = "^1.24.0"
 shap = "^0.42.0"
 statsmodels = "^0.13.5"
@@ -57,9 +52,9 @@ textstat = "^0.7.3"
 torch = {version = ">=1.10.0", optional = true}
 torchmetrics = {version = "^1.1.1", optional = true}
 tqdm = "^4.64.0"
-transformers = "^4.32.0"
-wget = "^3.2"
+transformers = {version = "^4.32.0", optional = true}
 xgboost = "^1.5.2"
+ydata-profiling = "^4.7.0"
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.1.0"
@@ -69,9 +64,10 @@ flake8 = "^4.0.1"
 gradio = "^3.43.2"
 ipykernel = "^6.22.0"
 isort = "^5.12.0"
-jupyterlab = "^3.6.3"
+jupyter = "^1.0.0"
 openai = "^1.3.7"
 papermill = "^2.4.0"
+pdoc = "^14.4.0"
 pre-commit = "^3.3.3"
 pytest = "^5.2"
 sphinx = "^6.1.3"
@@ -88,7 +84,6 @@ all = [
   "pycocoevalcap",
   "torchmetrics",
   "levenshtein",
-  "datasets",
 ]
 llm = [
   "torch",
@@ -97,7 +92,6 @@ llm = [
   "pycocoevalcap",
   "torchmetrics",
   "levenshtein",
-  "datasets",
 ]
 pytorch = ["torch"]
 r-support = ["rpy2"]
@@ -110,3 +104,6 @@ requires = ["poetry-core>=1.0.0"]
 [tool.isort]
 known_first_party = "validmind"
 profile = "black"
+
+[tool.poetry.scripts]
+vm-create-new-test = "scripts.create_new_test:generate_test"

{validmind-2.0.0 → validmind-2.0.7}/validmind/__init__.py

@@ -50,7 +50,7 @@ from .__version__ import __version__  # noqa: E402
 from .api_client import init
 from .api_client import log_figure as _log_figure_async
 from .api_client import log_metrics as _log_metrics_async
-from .api_client import log_test_results
+from .api_client import log_test_results, reload
 from .client import (  # noqa: E402
     get_test_suite,
     init_dataset,
@@ -60,6 +60,7 @@ from .client import (  # noqa: E402
     run_documentation_tests,
     run_test_suite,
 )
+from .tests.decorator import metric
 from .unit_metrics import run_metric
 from .utils import run_async  # noqa: E402
 
@@ -105,7 +106,9 @@ __all__ = [  # noqa
     "init_dataset",
     "init_model",
     "init_r_model",
+    "metric",
     "preview_template",
+    "reload",
     "run_documentation_tests",
     "run_test_suite",
     "tests",

validmind-2.0.7/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.0.7"

validmind-2.0.7/validmind/ai.py

@@ -0,0 +1,197 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import concurrent.futures
+import os
+
+from openai import AzureOpenAI, OpenAI
+
+from .utils import clean_docstring
+
+SYSTEM_PROMPT = """
+You are an expert data scientist and MRM specialist tasked with providing concise and'
+objective insights based on the results of quantitative model or dataset analysis.
+
+Examine the provided statistical test results and compose a brief summary. Highlight crucial
+insights, focusing on the distribution characteristics, central tendencies (such as mean or median),
+and the variability (including standard deviation and range) of the metrics. Evaluate how
+these statistics might influence the development and performance of a predictive model. Identify
+and explain any discernible trends or anomalies in the test results.
+
+Your analysis will act as the description of the result in the model documentation.
+
+Avoid long sentences and complex vocabulary.
+Structure the response clearly and logically.
+Use Markdown syntax to format the response.
+Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
+Use the following format for the response:
+```
+**<Test Name>** <continue to explain what it does in detail>...
+
+The results of this test <detailed explanation of the results>...
+
+In summary the following key insights can be gained from this <Test Type>
+
+- **<key insight 1 - title>**: <explanation of key insight 1>
+- ...<continue with any other key insights using the same format>
+```
+It is very important that the text is nicely formatted and contains enough information to be useful to the user as documentation.
+
+- use valid markdown syntax: make sure to have two newlines between paragraphs and before bullet points etc.
+""".strip()
+USER_PROMPT = """
+Test ID: {test_name}
+Test Description: {test_description}
+Test Results (the raw results of the test):
+{test_results}
+Test Summary (what the user sees in the documentation):
+{test_summary}
+""".strip()
+USER_PROMPT_FIGURES = """
+Test ID: {test_name}
+Test Description: {test_description}
+The attached plots show the results of the test.
+""".strip()
+
+__client = None
+__model = None
+
+__executor = concurrent.futures.ThreadPoolExecutor()
+
+
+def __get_client_and_model():
+    """
+    Get the model to use for generating interpretations
+    """
+    global __client, __model
+
+    if __client and __model:
+        return __client, __model
+
+    if "OPENAI_API_KEY" in os.environ:
+        __client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
+        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4-turbo-preview")
+
+    elif "AZURE_OPENAI_KEY" in os.environ:
+        if "AZURE_OPENAI_ENDPOINT" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_ENDPOINT must be set to run LLM tests with Azure"
+            )
+
+        if "AZURE_OPENAI_MODEL" not in os.environ:
+            raise ValueError(
+                "AZURE_OPENAI_MODEL must be set to run LLM tests with Azure"
+            )
+
+        __client = AzureOpenAI(
+            azure_endpoint=os.environ.get("AZURE_OPENAI_ENDPOINT"),
+            api_key=os.environ.get("AZURE_OPENAI_KEY"),
+            api_version=os.environ.get("AZURE_OPENAI_VERSION", "2023-05-15"),
+        )
+        __model = os.environ.get("AZURE_OPENAI_MODEL")
+
+    else:
+        raise ValueError("OPENAI_API_KEY or AZURE_OPENAI_KEY must be set")
+
+    return __client, __model
+
+
+class DescriptionFuture:
+    """This will be immediately returned from generate_description so that
+    the tests can continue to be run in parallel while the description is
+    retrieved asynchronously.
+
+    The value will be retrieved later and if its not ready yet, it should
+    block until it is.
+    """
+
+    def __init__(self, future):
+        self._future = future
+
+    def get_description(self):
+        # This will block until the future is completed
+        return clean_docstring(self._future.result())
+
+
+def generate_description_async(
+    test_name: str,
+    test_description: str,
+    test_results: str,
+    test_summary: str,
+    figures: list = None,
+):
+    """Generate the description for the test results"""
+    client, _ = __get_client_and_model()
+
+    # get last part of test id
+    test_name = test_name.split(".")[-1]
+
+    if not test_results and not test_summary:
+        if not figures:
+            raise ValueError("No results, summary or figures provided")
+
+        response = client.chat.completions.create(
+            model="gpt-4-1106-vision-preview",
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": [
+                        {
+                            "type": "text",
+                            "text": USER_PROMPT_FIGURES.format(
+                                test_name=test_name,
+                                test_description=test_description,
+                            ),
+                        },
+                        *[
+                            {
+                                "type": "image_url",
+                                "image_url": {
+                                    "url": figure._get_b64_url(),
+                                },
+                            }
+                            for figure in figures
+                        ],
+                    ],
+                },
+            ],
+        )
+    else:
+        response = client.chat.completions.create(
+            model="gpt-4-turbo-preview",
+            messages=[
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {
+                    "role": "user",
+                    "content": USER_PROMPT.format(
+                        test_name=test_name,
+                        test_description=test_description,
+                        test_results=test_results,
+                        test_summary=test_summary,
+                    ),
+                },
+            ],
+        )
+
+    return response.choices[0].message.content.strip("```").strip()
+
+
+def generate_description(
+    test_name: str,
+    test_description: str,
+    test_results: str,
+    test_summary: str,
+    figures: list = None,
+):
+    future = __executor.submit(
+        generate_description_async,
+        test_name,
+        test_description,
+        test_results,
+        test_summary,
+        figures,
+    )
+
+    return DescriptionFuture(future)

{validmind-2.0.0 → validmind-2.0.7}/validmind/api_client.py

@@ -171,6 +171,18 @@ def __ping() -> Dict[str, Any]:
     )
 
 
+def reload():
+    """Reconnect to the ValidMind API and reload the project configuration"""
+
+    try:
+        __ping()
+    except Exception as e:
+        # if the api host is https, assume we're not in dev mode and send to sentry
+        if _api_host.startswith("https://"):
+            send_single_error(e)
+        raise e
+
+
 async def __get_url(endpoint: str, params: Optional[Dict[str, str]] = None) -> str:
     if not _run_cuid:
         start_run()
@@ -313,14 +325,14 @@ async def log_figures(figures: List[Figure]) -> Dict[str, Any]:
 async def log_metadata(
     content_id: str,
     text: Optional[str] = None,
-    extra_json: Optional[Dict[str, Any]] = None,
+    _json: Optional[Dict[str, Any]] = None,
 ) -> Dict[str, Any]:
     """Logs free-form metadata to ValidMind API.
 
     Args:
         content_id (str): Unique content identifier for the metadata
         text (str, optional): Free-form text to assign to the metadata. Defaults to None.
-        extra_json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
+        _json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
 
     Raises:
         Exception: If the API call fails
@@ -331,8 +343,8 @@ async def log_metadata(
     metadata_dict = {"content_id": content_id}
     if text is not None:
         metadata_dict["text"] = text
-    if extra_json is not None:
-        metadata_dict["extra_json"] = extra_json
+    if _json is not None:
+        metadata_dict["json"] = _json
 
     try:
         return await _post(

{validmind-2.0.0 → validmind-2.0.7}/validmind/client.py

@@ -7,6 +7,7 @@ Client interface for all data and model validation functions
 """
 
 import pandas as pd
+import polars as pl
 
 from .api_client import _log_input as log_input
 from .client_config import client_config
@@ -26,7 +27,13 @@ from .template import preview_template as _preview_template
 from .test_suites import get_by_id as get_test_suite_by_id
 from .utils import get_dataset_info, get_model_info
 from .vm_models import TestInput, TestSuite, TestSuiteRunner
-from .vm_models.dataset import DataFrameDataset, NumpyDataset, TorchDataset, VMDataset
+from .vm_models.dataset import (
+    DataFrameDataset,
+    NumpyDataset,
+    PolarsDataset,
+    TorchDataset,
+    VMDataset,
+)
 from .vm_models.model import VMModel, get_model_class
 
 pd.option_context("format.precision", 2)
@@ -58,7 +65,7 @@ def init_dataset(
     DataFrames at the moment.
 
     Args:
-        dataset (pd.DataFrame): We only support Pandas DataFrames at the moment
+        dataset : dataset from various python libraries
         model (VMModel): ValidMind model object
         options (dict): A dictionary of options for the dataset
         targets (vm.vm.DatasetTargets): A list of target variables
@@ -89,7 +96,7 @@ def init_dataset(
     input_id = input_id or "dataset"
 
     # Instantiate supported dataset types here
-    if dataset_class == "DataFrame":
+    if isinstance(dataset, pd.DataFrame):
         logger.info("Pandas dataset detected. Initializing VM Dataset instance...")
         vm_dataset = DataFrameDataset(
             input_id=input_id,
@@ -102,6 +109,19 @@ def init_dataset(
             target_class_labels=class_labels,
             date_time_index=date_time_index,
         )
+    elif isinstance(dataset, pl.DataFrame):
+        logger.info("Polars dataset detected. Initializing VM Dataset instance...")
+        vm_dataset = PolarsDataset(
+            input_id=input_id,
+            raw_dataset=dataset,
+            model=model,
+            target_column=target_column,
+            feature_columns=feature_columns,
+            text_column=text_column,
+            extra_columns=extra_columns,
+            target_class_labels=class_labels,
+            date_time_index=date_time_index,
+        )
     elif dataset_class == "ndarray":
         logger.info("Numpy ndarray detected. Initializing VM Dataset instance...")
         vm_dataset = NumpyDataset(

{validmind-2.0.0 → validmind-2.0.7}/validmind/datasets/classification/customer_churn.py

@@ -60,7 +60,7 @@ def preprocess(df):
     return train_df, validation_df, test_df
 
 
-def get_demo_test_config():
+def get_demo_test_config(test_suite=None):
     """
     Returns input configuration for the default documentation
     template assigned to this demo model
@@ -81,7 +81,7 @@ def get_demo_test_config():
     - The only exception is ClassifierPerformance since that runs twice: once
         with the train_dataset (in sample) and once with the test_dataset (out of sample)
     """
-    default_config = vm.get_test_suite().get_default_config()
+    default_config = (test_suite or vm.get_test_suite()).get_default_config()
 
     for _, test_config in default_config.items():
         if "model" in test_config["inputs"]:

{validmind-2.0.0 → validmind-2.0.7}/validmind/datasets/nlp/__init__.py

@@ -5,3 +5,8 @@
 """
 Example datasets that can be used with the developer framework.
 """
+
+__all__ = [
+    "cnn_dailymail",
+    "twitter_covide_19",
+]

validmind-2.0.7/validmind/datasets/nlp/cnn_dailymail.py

@@ -0,0 +1,98 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import os
+import textwrap
+
+import pandas as pd
+from datasets import load_dataset
+from IPython.display import HTML, display
+from tabulate import tabulate
+
+# Define column names
+text_column = "article"
+target_column = "highlights"
+gpt_35_prediction_column = "gpt_35_prediction"
+t5_prediction = "t5_prediction"
+
+# Define the path to the dataset directory
+current_path = os.path.dirname(os.path.abspath(__file__))
+dataset_path = os.path.join(current_path, "datasets")
+
+
+def load_data(source="online", dataset_size=None):
+    """
+    Load data from either online source or offline files.
+
+    :param source: 'online' for online data, 'offline' for offline data. Defaults to 'online'.
+    :param dataset_size: Applicable if source is 'offline'. '300k' or '500k' for dataset size. Defaults to None.
+    :return: DataFrame containing the loaded data.
+    """
+    if source == "online":
+        # Load online data without predictions
+        cnn_dataset = load_dataset("cnn_dailymail", "3.0.0")
+        train_df = cnn_dataset["train"].to_pandas()
+        test_df = cnn_dataset["test"].to_pandas()
+
+        # Process the DataFrame to include necessary columns
+        train_df = train_df[["article", "highlights"]]
+        test_df = test_df[["article", "highlights"]]
+
+        return train_df, test_df
+
+    elif source == "offline":
+        # Determine the file name based on the dataset size
+        if dataset_size == "100":
+            data_file_name = "cnn_dailymail_100_with_predictions.csv"
+        elif dataset_size == "500":
+            data_file_name = "cnn_dailymail_500_with_predictions.csv"
+        else:
+            raise ValueError("Invalid dataset_size specified. Choose '100' or '500'.")
+
+        # Construct the file path
+        data_file = os.path.join(dataset_path, data_file_name)
+
+        # Load the dataset
+        df = pd.read_csv(data_file)
+        df = df[["article", "highlights", "gpt_35_prediction", "t5_prediction"]]
+
+        train_df = df.sample(frac=0.7, random_state=42)
+        test_df = df.drop(train_df.index)
+        return train_df, test_df
+
+    else:
+        raise ValueError("Invalid source specified. Choose 'online' or 'offline'.")
+
+
+def _format_cell_text(text, width=50):
+    """Private function to format a cell's text."""
+    return "\n".join([textwrap.fill(line, width=width) for line in text.split("\n")])
+
+
+def _format_dataframe_for_tabulate(df):
+    """Private function to format the entire DataFrame for tabulation."""
+    df_out = df.copy()
+
+    # Format all string columns
+    for column in df_out.columns:
+        # Check if column is of type object (likely strings)
+        if df_out[column].dtype == object:
+            df_out[column] = df_out[column].apply(_format_cell_text)
+    return df_out
+
+
+def _dataframe_to_html_table(df):
+    """Private function to convert a DataFrame to an HTML table."""
+    headers = df.columns.tolist()
+    table_data = df.values.tolist()
+    return tabulate(table_data, headers=headers, tablefmt="html")
+
+
+def display_nice(df, num_rows=None):
+    """Primary function to format and display a DataFrame."""
+    if num_rows is not None:
+        df = df.head(num_rows)
+    formatted_df = _format_dataframe_for_tabulate(df)
+    html_table = _dataframe_to_html_table(formatted_df)
+    display(HTML(html_table))