validmind 2.4.7__tar.gz → 2.4.10__tar.gz

{validmind-2.4.7 → validmind-2.4.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.4.7
+Version: 2.4.10
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
@@ -29,6 +29,7 @@ Requires-Dist: latex2mathml (>=3.77.0)
 Requires-Dist: llvmlite ; python_version >= "3.8" and python_full_version <= "3.11.0"
 Requires-Dist: matplotlib
 Requires-Dist: mistune (>=3.0.2,<4.0.0)
+Requires-Dist: nest-asyncio (>=1.6.0,<2.0.0)
 Requires-Dist: nltk (>=3.8.1,<4.0.0)
 Requires-Dist: numba (<0.59.0)
 Requires-Dist: numpy

{validmind-2.4.7 → validmind-2.4.10}/pyproject.toml

@@ -10,9 +10,10 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.4.7"
+version = "2.4.10"
 
 [tool.poetry.dependencies]
+python = ">=3.8.1,<3.12"
 aiohttp = {extras = ["speedups"], version = "*"}
 arch = "*"
 bert-score = ">=0.3.13"
@@ -26,6 +27,7 @@ latex2mathml = ">=3.77.0"
 llvmlite = {version = "*", python = ">=3.8,<=3.11"}
 matplotlib = "*"
 mistune = "^3.0.2"
+nest-asyncio = "^1.6.0"
 nltk = "^3.8.1"
 numba = "<0.59.0" # TODO: https://github.com/validmind/developer-framework/pull/28
 numpy = "*"
@@ -35,7 +37,6 @@ plotly = "*"
 plotly-express = "*"
 polars = "*"
 pycocoevalcap = {version = "^1.2", optional = true}
-python = ">=3.8.1,<3.12"
 python-dotenv = "*"
 ragas = {version = ">=0.1.7", optional = true}
 rouge = ">=1"

validmind-2.4.10/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.4.10"

{validmind-2.4.7 → validmind-2.4.10}/validmind/api_client.py

@@ -34,6 +34,7 @@ _api_secret = os.getenv("VM_API_SECRET")
 _api_host = os.getenv("VM_API_HOST")
 _model_cuid = os.getenv("VM_API_MODEL")
 _run_cuid = os.getenv("VM_RUN_CUID")
+_monitoring = False
 
 __api_session: aiohttp.ClientSession = None
 
@@ -57,6 +58,7 @@ def get_api_config() -> Dict[str, Optional[str]]:
         "VM_API_HOST": _api_host,
         "VM_API_MODEL": _model_cuid,
         "VM_RUN_CUID": _run_cuid,
+        "X-MONITORING": _monitoring,
     }
 
 
@@ -73,6 +75,7 @@ def get_api_headers() -> Dict[str, str]:
         "X-API-KEY": _api_key,
         "X-API-SECRET": _api_secret,
         "X-PROJECT-CUID": _model_cuid,
+        "X-MONITORING": _monitoring,
     }
 
 
@@ -82,6 +85,7 @@ def init(
     api_secret: Optional[str] = None,
     api_host: Optional[str] = None,
     model: Optional[str] = None,
+    monitoring=False,
 ):
     """
     Initializes the API client instances and calls the /ping endpoint to ensure
@@ -96,11 +100,12 @@ def init(
         api_key (str, optional): The API key. Defaults to None.
         api_secret (str, optional): The API secret. Defaults to None.
         api_host (str, optional): The API host. Defaults to None.
+        monitoring (str, optional): The ongoing monitoring flag. Defaults to False.
 
     Raises:
         ValueError: If the API key and secret are not provided
     """
-    global _api_key, _api_secret, _api_host, _run_cuid, _model_cuid
+    global _api_key, _api_secret, _api_host, _run_cuid, _model_cuid, _monitoring
 
     if api_key == "...":
         # special case to detect when running a notebook with the standard init snippet
@@ -125,6 +130,8 @@ def init(
 
     _run_cuid = os.getenv("VM_RUN_CUID", None)
 
+    _monitoring = monitoring
+
     try:
         __ping()
     except Exception as e:
@@ -159,6 +166,7 @@ def __ping() -> Dict[str, Any]:
             "X-API-KEY": _api_key,
             "X-API-SECRET": _api_secret,
             "X-PROJECT-CUID": _model_cuid,
+            "X-MONITORING": str(_monitoring),
         },
     )
     if r.status_code != 200:

{validmind-2.4.7 → validmind-2.4.10}/validmind/client.py

@@ -135,7 +135,8 @@ def init_dataset(
             model=model,
             index=index,
             index_name=index_name,
-            columns=columns,
+            # if no columns are passed, use the index
+            columns=columns or [i for i in range(dataset.shape[1])],
             target_column=target_column,
             feature_columns=feature_columns,
             text_column=text_column,

{validmind-2.4.7 → validmind-2.4.10}/validmind/html_templates/content_blocks.py

@@ -27,7 +27,8 @@ test_content_block_html = """
   <div id="expandable_instructions_{uuid}" style="display: {instructions_display};">
   <h4>Code:</h4>
     <pre>
-        <code class='language-python'>import validmind as vm
+        <code class='language-python'>
+import validmind as vm
 
 # inputs dictionary maps your inputs to the expected input names
 # keys are the expected input names and values are the actual inputs

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/data_validation/DatasetDescription.py

@@ -161,7 +161,7 @@ class DatasetDescription(Metric):
                 .describe(percentiles=[0.25, 0.5, 0.75, 0.9, 0.95])
                 .to_dict()
             )
-        elif field_type == "Categorical":
+        elif field_type == "Categorical" or field_type == "Text":
             field["statistics"] = (
                 df[field["id"]].astype("category").describe().to_dict()
             )

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/AnswerCorrectness.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import answer_correctness
 
 from validmind import tags, tasks
 
@@ -89,6 +87,12 @@ def AnswerCorrectness(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import answer_correctness
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/AnswerRelevance.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import answer_relevancy
 
 from validmind import tags, tasks
 
@@ -93,6 +91,12 @@ def AnswerRelevance(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import answer_relevancy
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/AnswerSimilarity.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import answer_similarity
 
 from validmind import tags, tasks
 
@@ -79,6 +77,12 @@ def AnswerSimilarity(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import answer_similarity
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/AspectCritique.py

@@ -6,28 +6,11 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics.critique import AspectCritique as _AspectCritique
-from ragas.metrics.critique import (
-    coherence,
-    conciseness,
-    correctness,
-    harmfulness,
-    maliciousness,
-)
 
 from validmind import tags, tasks
 
 from .utils import get_ragas_config, get_renamed_columns
 
-aspect_map = {
-    "coherence": coherence,
-    "conciseness": conciseness,
-    "correctness": correctness,
-    "harmfulness": harmfulness,
-    "maliciousness": maliciousness,
-}
-
 
 @tags("ragas", "llm", "qualitative")
 @tasks("text_summarization", "text_generation", "text_qa")
@@ -116,6 +99,27 @@ def AspectCritique(
     )
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics.critique import AspectCritique as _AspectCritique
+        from ragas.metrics.critique import (
+            coherence,
+            conciseness,
+            correctness,
+            harmfulness,
+            maliciousness,
+        )
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
+    aspect_map = {
+        "coherence": coherence,
+        "conciseness": conciseness,
+        "correctness": correctness,
+        "harmfulness": harmfulness,
+        "maliciousness": maliciousness,
+    }
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/ContextEntityRecall.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import context_entity_recall
 
 from validmind import tags, tasks
 
@@ -85,6 +83,12 @@ def ContextEntityRecall(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import context_entity_recall
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/ContextPrecision.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import context_precision
 
 from validmind import tags, tasks
 
@@ -80,6 +78,12 @@ def ContextPrecision(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import context_precision
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/ContextRecall.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import context_recall
 
 from validmind import tags, tasks
 
@@ -80,6 +78,12 @@ def ContextRecall(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import context_recall
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/ContextRelevancy.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import context_relevancy
 
 from validmind import tags, tasks
 
@@ -74,6 +72,12 @@ def ContextRelevancy(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import context_relevancy
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/Faithfulness.py

@@ -6,8 +6,6 @@ import warnings
 
 import plotly.express as px
 from datasets import Dataset
-from ragas import evaluate
-from ragas.metrics import faithfulness
 
 from validmind import tags, tasks
 
@@ -79,6 +77,12 @@ def Faithfulness(
     }
     ```
     """
+    try:
+        from ragas import evaluate
+        from ragas.metrics import faithfulness
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     warnings.filterwarnings(
         "ignore",
         category=FutureWarning,

{validmind-2.4.7 → validmind-2.4.10}/validmind/tests/model_validation/ragas/utils.py

@@ -4,14 +4,18 @@
 
 import os
 
-from langchain_openai import ChatOpenAI, OpenAIEmbeddings
-
 from validmind.ai.utils import get_client_and_model
 
 EMBEDDINGS_MODEL = "text-embedding-3-small"
 
 
 def get_ragas_config():
+    # import here since its an optional dependency
+    try:
+        from langchain_openai import ChatOpenAI, OpenAIEmbeddings
+    except ImportError:
+        raise ImportError("Please run `pip install validmind[llm]` to use LLM tests")
+
     client, model = get_client_and_model()
     os.environ["OPENAI_API_BASE"] = str(client.base_url)
 

{validmind-2.4.7 → validmind-2.4.10}/validmind/vm_models/dataset/dataset.py

@@ -92,14 +92,14 @@ class VMDataset:
             raise ValueError("Expected Numpy array for attribute raw_dataset")
         self.index = index
 
-        self.df = pd.DataFrame(self._raw_dataset, columns=columns).infer_objects()
+        self._df = pd.DataFrame(self._raw_dataset, columns=columns).infer_objects()
         # set index to dataframe
         if index is not None:
-            self.df.set_index(pd.Index(index), inplace=True)
-            self.df.index.name = index_name
+            self._df.set_index(pd.Index(index), inplace=True)
+            self._df.index.name = index_name
         # attempt to convert index to datatime
         if date_time_index:
-            self.df = convert_index_to_datetime(self.df)
+            self._df = convert_index_to_datetime(self._df)
 
         self.options = options
 
@@ -128,12 +128,12 @@ class VMDataset:
             self.feature_columns = [col for col in self.columns if col not in excluded]
 
         self.feature_columns_numeric = (
-            self.df[self.feature_columns]
+            self._df[self.feature_columns]
             .select_dtypes(include=[np.number])
             .columns.tolist()
         )
         self.feature_columns_categorical = (
-            self.df[self.feature_columns]
+            self._df[self.feature_columns]
             .select_dtypes(include=[object, pd.Categorical])
             .columns.tolist()
         )
@@ -142,19 +142,19 @@ class VMDataset:
         column_values = np.array(column_values)
 
         if column_values.ndim == 1:
-            if len(column_values) != len(self.df):
+            if len(column_values) != len(self._df):
                 raise ValueError(
                     "Length of values doesn't match number of rows in the DataFrame."
                 )
             self.columns.append(column_name)
-            self.df[column_name] = column_values
+            self._df[column_name] = column_values
         elif column_values.ndim == 2:
-            if column_values.shape[0] != len(self.df):
+            if column_values.shape[0] != len(self._df):
                 raise ValueError(
                     "Number of rows in values doesn't match number of rows in the DataFrame."
                 )
             self.columns.append(column_name)
-            self.df[column_name] = column_values.tolist()
+            self._df[column_name] = column_values.tolist()
 
         else:
             raise ValueError("Only 1D and 2D arrays are supported for column_values.")
@@ -243,10 +243,10 @@ class VMDataset:
 
         # if the user passes a column name, we assume it has precomputed predictions
         if prediction_column:
-            prediction_values = self.df[prediction_column].values
+            prediction_values = self._df[prediction_column].values
 
             if probability_column:
-                probability_values = self.df[probability_column].values
+                probability_values = self._df[probability_column].values
 
         if prediction_values is None:
             X = self.df if isinstance(model, (FunctionModel, PipelineModel)) else self.x
@@ -320,6 +320,33 @@ class VMDataset:
             f"Extra column {column_name} with {len(column_values)} values added to the dataset"
         )
 
+    @property
+    def df(self) -> pd.DataFrame:
+        """
+        Returns the dataset as a pandas DataFrame.
+
+        Returns:
+            pd.DataFrame: The dataset as a pandas DataFrame.
+        """
+        # only include feature, text and target columns
+        # don't include internal pred and prob columns
+        columns = self.feature_columns.copy()
+
+        # text column can also be a feature column so don't add it twice
+        if self.text_column and self.text_column not in columns:
+            columns.append(self.text_column)
+
+        if self.extra_columns.extras:
+            # add user-defined extra columns
+            columns.extend(self.extra_columns.extras)
+
+        if self.target_column:
+            # shouldn't be a feature column but add this to be safe
+            assert self.target_column not in columns
+            columns.append(self.target_column)
+
+        return as_df(self._df[columns])
+
     @property
     def x(self) -> np.ndarray:
         """
@@ -328,7 +355,7 @@ class VMDataset:
         Returns:
             np.ndarray: The input features.
         """
-        return self.df[self.feature_columns].to_numpy()
+        return self._df[self.feature_columns].to_numpy()
 
     @property
     def y(self) -> np.ndarray:
@@ -338,7 +365,7 @@ class VMDataset:
         Returns:
             np.ndarray: The target variables.
         """
-        return self.df[self.target_column].to_numpy()
+        return self._df[self.target_column].to_numpy()
 
     def y_pred(self, model) -> np.ndarray:
         """Returns the predictions for a given model.
@@ -352,7 +379,7 @@ class VMDataset:
         Returns:
             np.ndarray: The predictions for the model
         """
-        return np.stack(self.df[self.prediction_column(model)].values)
+        return np.stack(self._df[self.prediction_column(model)].values)
 
     def y_prob(self, model) -> np.ndarray:
         """Returns the probabilities for a given model.
@@ -363,23 +390,23 @@ class VMDataset:
         Returns:
             np.ndarray: The probability variables.
         """
-        return self.df[self.probability_column(model)].values
+        return self._df[self.probability_column(model)].values
 
     def x_df(self):
         """Returns a dataframe containing only the feature columns"""
-        return as_df(self.df[self.feature_columns])
+        return as_df(self._df[self.feature_columns])
 
     def y_df(self) -> pd.DataFrame:
         """Returns a dataframe containing the target column"""
-        return as_df(self.df[self.target_column])
+        return as_df(self._df[self.target_column])
 
     def y_pred_df(self, model) -> pd.DataFrame:
         """Returns a dataframe containing the predictions for a given model"""
-        return as_df(self.df[self.prediction_column(model)])
+        return as_df(self._df[self.prediction_column(model)])
 
     def y_prob_df(self, model) -> pd.DataFrame:
         """Returns a dataframe containing the probabilities for a given model"""
-        return as_df(self.df[self.probability_column(model)])
+        return as_df(self._df[self.probability_column(model)])
 
     def target_classes(self):
         """Returns the target class labels or unique values of the target column."""

{validmind-2.4.7 → validmind-2.4.10}/validmind/vm_models/dataset/utils.py

@@ -16,6 +16,8 @@ logger = get_logger(__name__)
 
 @dataclass
 class ExtraColumns:
+    # TODO: this now holds internal (pred, prob and group_by) cols as well as
+    # user-defined extra columns. These should probably be separated.
     """Extra columns for the dataset."""
 
     extras: Set[str] = field(default_factory=set)

validmind-2.4.7/validmind/__version__.py

@@ -1 +0,0 @@
-__version__ = "2.4.7"