validmind 2.10.0rc2__tar.gz → 2.11.0__tar.gz

{validmind-2.10.0rc2 → validmind-2.11.0}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: validmind
-Version: 2.10.0rc2
+Version: 2.11.0
 Summary: ValidMind Library
 License: DUAL LICENSE NOTICE
          
@@ -689,6 +689,7 @@ License: DUAL LICENSE NOTICE
          if any, to sign a "copyright disclaimer" for the program, if necessary.
          For more information on this, and how to apply and follow the GNU AGPL, see
          <https://www.gnu.org/licenses/>.
+License-File: LICENSE
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
 Requires-Python: >=3.9,<3.13
@@ -705,6 +706,7 @@ Provides-Extra: explainability
 Provides-Extra: huggingface
 Provides-Extra: llm
 Provides-Extra: nlp
+Provides-Extra: pii-detection
 Provides-Extra: pytorch
 Provides-Extra: stats
 Provides-Extra: xgboost
@@ -716,6 +718,7 @@ Requires-Dist: beautifulsoup4
 Requires-Dist: bert-score (>=0.3.13) ; extra == "all"
 Requires-Dist: bert-score (>=0.3.13) ; extra == "nlp"
 Requires-Dist: datasets (>=2.10.0,<3.0.0) ; extra == "datasets"
+Requires-Dist: deepeval (>3.3.9) ; extra == "llm"
 Requires-Dist: evaluate ; extra == "all"
 Requires-Dist: evaluate ; extra == "nlp"
 Requires-Dist: ipywidgets
@@ -729,10 +732,14 @@ Requires-Dist: mistune (>=3.0.2,<4.0.0)
 Requires-Dist: nest-asyncio (>=1.6.0,<2.0.0)
 Requires-Dist: nltk (>=3.8.1,<4.0.0) ; extra == "all"
 Requires-Dist: nltk (>=3.8.1,<4.0.0) ; extra == "nlp"
+Requires-Dist: numpy (>=1.23,<2.0.0)
 Requires-Dist: openai (>=1)
 Requires-Dist: pandas (>=2.0.3,<3.0.0)
-Requires-Dist: plotly (>=5.0.0)
+Requires-Dist: plotly (>=5.0.0,<6.0.0)
 Requires-Dist: polars
+Requires-Dist: presidio-analyzer ; extra == "pii-detection"
+Requires-Dist: presidio-structured ; extra == "pii-detection"
+Requires-Dist: pyarrow (<16) ; extra == "nlp"
 Requires-Dist: pycocoevalcap ; extra == "all"
 Requires-Dist: pycocoevalcap ; extra == "llm"
 Requires-Dist: python-dotenv
@@ -749,12 +756,11 @@ Requires-Dist: seaborn
 Requires-Dist: sentencepiece (>=0.2.0,<0.3.0) ; extra == "all"
 Requires-Dist: sentencepiece (>=0.2.0,<0.3.0) ; extra == "huggingface"
 Requires-Dist: sentencepiece (>=0.2.0,<0.3.0) ; extra == "llm"
-Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: shap (>=0.46.0) ; extra == "all"
 Requires-Dist: shap (>=0.46.0) ; extra == "explainability"
 Requires-Dist: statsmodels ; extra == "all"
 Requires-Dist: statsmodels ; extra == "stats"
-Requires-Dist: tabulate (>=0.8.9,<0.9.0)
+Requires-Dist: tabulate (>=0.9.0,<0.10.0)
 Requires-Dist: textblob (>=0.18.0.post0,<0.19.0) ; extra == "all"
 Requires-Dist: textblob (>=0.18.0.post0,<0.19.0) ; extra == "nlp"
 Requires-Dist: tiktoken

{validmind-2.10.0rc2 → validmind-2.11.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.10.0-rc.2"
+version = "2.11.0"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.13"
@@ -19,15 +19,15 @@ dependencies = [
   "matplotlib",
   "mistune (>=3.0.2,<4.0.0)",
   "nest-asyncio (>=1.6.0,<2.0.0)",
+  "numpy (>=1.23,<2.0.0)",
   "openai (>=1)",
   "pandas (>=2.0.3,<3.0.0)",
-  "plotly (>=5.0.0)",
+  "plotly (>=5.0.0,<6.0.0)",
   "polars",
   "python-dotenv",
   "scikit-learn",
   "seaborn",
-  "sentry-sdk (>=1.24.0,<2.0.0)",
-  "tabulate (>=0.8.9,<0.9.0)",
+  "tabulate (>=0.9.0,<0.10.0)",
   "tiktoken",
   "tqdm",
   "anywidget",
@@ -66,6 +66,7 @@ llm = [
   "ragas (>=0.2.3,<=0.2.7)",
   "sentencepiece (>=0.2.0,<0.3.0)",
   "langchain-openai (>=0.1.8)",
+  "deepeval (>3.3.9)",
 ]
 nlp = [
   "langdetect",
@@ -74,6 +75,7 @@ nlp = [
   "evaluate",
   "rouge (>=1)",
   "bert-score (>=0.3.13)",
+  "pyarrow (<16)",
 ]
 pytorch = ["torch (>=2.0.0)"]
 stats = ["scipy", "statsmodels", "arch"]
@@ -81,13 +83,14 @@ xgboost = ["xgboost (>=1.5.2,<3)"]
 explainability = ["shap (>=0.46.0)"]
 credit_risk = ["scorecardpy (>=0.1.9.6,<0.2.0)"]
 datasets = ["datasets (>=2.10.0,<3.0.0)"]
+pii-detection = ["presidio-analyzer", "presidio-structured"]
 
 [tool.poetry.group.dev.dependencies]
 black = "^22.1.0"
 click = "*"
 cython = "^0.29.34"
 docstring_parser = "*"
-flake8 = "^4.0.1"
+flake8 = "^7.0.0"
 griffe = "*"
 ipykernel = "^6.22.0"
 isort = "^5.12.0"

{validmind-2.10.0rc2 → validmind-2.11.0}/validmind/__init__.py

@@ -21,7 +21,7 @@ To initialize the ValidMind Library, paste the code snippet with the model ident
 import validmind as vm
 
 vm.init(
-  api_host = "https://api.dev.vm.validmind.ai/api/v1/tracking/tracking",
+  api_host = "https://app.prod.validmind.ai/api/v1/tracking/tracking",
   api_key = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
   api_secret = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
   project = "<project-identifier>"
@@ -48,6 +48,7 @@ try:
 except ImportError:
     ...
 
+from . import scorer
 from .__version__ import __version__  # noqa: E402
 from .api_client import init, log_metric, log_text, reload
 from .client import (  # noqa: E402
@@ -60,6 +61,7 @@ from .client import (  # noqa: E402
     run_test_suite,
 )
 from .experimental import agents as experimental_agent
+from .tests.decorator import scorer as scorer_decorator
 from .tests.decorator import tags, tasks, test
 from .tests.run import print_env
 from .utils import is_notebook, parse_version
@@ -128,6 +130,9 @@ __all__ = [  # noqa
     "tags",
     "tasks",
     "test",
+    "scorer_decorator",
+    # scorer module
+    "scorer",
     # raw data (for post-processing test results and building tests)
     "RawData",
     # submodules

validmind-2.11.0/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.11.0"

{validmind-2.10.0rc2 → validmind-2.11.0}/validmind/ai/test_descriptions.py

@@ -5,21 +5,39 @@
 import json
 import os
 from concurrent.futures import ThreadPoolExecutor
-from typing import List, Optional, Union
-
-import tiktoken
+from typing import Any, Dict, List, Optional, Union
 
 from ..client_config import client_config
 from ..logging import get_logger
 from ..utils import NumpyEncoder, md_to_html, test_id_to_name
 from ..vm_models.figure import Figure
 from ..vm_models.result import ResultTable
+from ..vm_models.result.pii_filter import (
+    PIIDetectionMode,
+    get_pii_detection_mode,
+    scan_df,
+)
 from .utils import DescriptionFuture
 
 __executor = ThreadPoolExecutor()
 
 logger = get_logger(__name__)
 
+# Try to import tiktoken once at module load
+# Cache the result to avoid repeated import attempts
+_TIKTOKEN_AVAILABLE = False
+_TIKTOKEN_ENCODING = None
+
+try:
+    import tiktoken
+
+    _TIKTOKEN_ENCODING = tiktoken.encoding_for_model("gpt-4o")
+    _TIKTOKEN_AVAILABLE = True
+except (ImportError, Exception) as e:
+    logger.debug(
+        f"tiktoken unavailable, will use character-based token estimation: {e}"
+    )
+
 
 def _get_llm_global_context():
     # Get the context from the environment variable
@@ -37,6 +55,29 @@ def _get_llm_global_context():
     return context if context_enabled and context else None
 
 
+def _estimate_tokens_simple(text: str) -> int:
+    """Estimate token count using character-based heuristic.
+
+    Uses ~4 characters per token for English/JSON text.
+    This is a fallback when tiktoken is unavailable.
+    """
+    return len(text) // 4
+
+
+def _truncate_text_simple(text: str, max_tokens: int) -> str:
+    """Truncate text using character-based estimation."""
+    estimated_chars = max_tokens * 4
+    if len(text) <= estimated_chars:
+        return text
+
+    # Keep first portion and last 100 tokens worth (~400 chars)
+    # But ensure we don't take more than 25% for the tail
+    last_chars = min(400, estimated_chars // 4)
+    first_chars = estimated_chars - last_chars
+
+    return text[:first_chars] + "...[truncated]" + text[-last_chars:]
+
+
 def _truncate_summary(
     summary: Union[str, None], test_id: str, max_tokens: int = 100_000
 ):
@@ -44,20 +85,30 @@ def _truncate_summary(
         # since string itself is less than max_tokens, definitely small enough
         return summary
 
-    # TODO: better context length handling
-    encoding = tiktoken.encoding_for_model("gpt-4o")
-    summary_tokens = encoding.encode(summary)
+    if _TIKTOKEN_AVAILABLE:
+        # Use tiktoken for accurate token counting
+        summary_tokens = _TIKTOKEN_ENCODING.encode(summary)
 
-    if len(summary_tokens) > max_tokens:
-        logger.warning(
-            f"Truncating {test_id} due to context length restrictions..."
-            " Generated description may be innacurate"
-        )
-        summary = (
-            encoding.decode(summary_tokens[:max_tokens])
-            + "...[truncated]"
-            + encoding.decode(summary_tokens[-100:])
-        )
+        if len(summary_tokens) > max_tokens:
+            logger.warning(
+                f"Truncating {test_id} due to context length restrictions..."
+                " Generated description may be inaccurate"
+            )
+            summary = (
+                _TIKTOKEN_ENCODING.decode(summary_tokens[:max_tokens])
+                + "...[truncated]"
+                + _TIKTOKEN_ENCODING.decode(summary_tokens[-100:])
+            )
+    else:
+        # Fallback to character-based estimation
+        estimated_tokens = _estimate_tokens_simple(summary)
+
+        if estimated_tokens > max_tokens:
+            logger.warning(
+                f"Truncating {test_id} (estimated) due to context length restrictions..."
+                " Generated description may be inaccurate"
+            )
+            summary = _truncate_text_simple(summary, max_tokens)
 
     return summary
 
@@ -69,6 +120,9 @@ def generate_description(
     metric: Union[float, int] = None,
     figures: List[Figure] = None,
     title: Optional[str] = None,
+    instructions: Optional[str] = None,
+    additional_context: Optional[str] = None,
+    params: Optional[Dict[str, Any]] = None,
 ):
     """Generate the description for the test results."""
     from validmind.api_client import generate_test_result_description
@@ -92,6 +146,13 @@ def generate_description(
         )
 
     if tables:
+        if get_pii_detection_mode() in [
+            PIIDetectionMode.TEST_DESCRIPTIONS,
+            PIIDetectionMode.ALL,
+        ]:
+            for table in tables:
+                scan_df(table.data)
+
         summary = "\n---\n".join(
             [
                 json.dumps(table.serialize(), cls=NumpyEncoder, separators=(",", ":"))
@@ -107,10 +168,10 @@ def generate_description(
             "test_description": test_description,
             "title": title,
             "summary": _truncate_summary(summary, test_id),
-            "figures": [
-                figure._get_b64_url() for figure in ([] if tables else figures)
-            ],
-            "context": _get_llm_global_context(),
+            "figures": [figure._get_b64_url() for figure in figures or []],
+            "additional_context": additional_context,
+            "instructions": instructions,
+            "params": params,
         }
     )["content"]
 
@@ -122,16 +183,25 @@ def background_generate_description(
     figures: List[Figure] = None,
     metric: Union[int, float] = None,
     title: Optional[str] = None,
+    instructions: Optional[str] = None,
+    additional_context: Optional[str] = None,
+    params: Optional[Dict[str, Any]] = None,
 ):
     def wrapped():
         try:
-            return generate_description(
-                test_id=test_id,
-                test_description=test_description,
-                tables=tables,
-                figures=figures,
-                metric=metric,
-                title=title,
+            return (
+                generate_description(
+                    test_id=test_id,
+                    test_description=test_description,
+                    tables=tables,
+                    figures=figures,
+                    metric=metric,
+                    title=title,
+                    instructions=instructions,
+                    additional_context=additional_context,
+                    params=params,
+                ),
+                True,
             )
         except Exception as e:
             if "maximum context length" in str(e):
@@ -146,7 +216,7 @@ def background_generate_description(
                 logger.warning(f"Failed to generate description for {test_id}: {e}")
             logger.warning(f"Using default description for {test_id}")
 
-            return test_description
+            return test_description, False
 
     return DescriptionFuture(__executor.submit(wrapped))
 
@@ -159,6 +229,9 @@ def get_result_description(
     metric: Union[int, float] = None,
     should_generate: bool = True,
     title: Optional[str] = None,
+    instructions: Optional[str] = None,
+    additional_context: Optional[str] = None,
+    params: Optional[Dict[str, Any]] = None,
 ):
     """Get the metadata dictionary for a test or metric result.
 
@@ -180,10 +253,18 @@ def get_result_description(
         figures (List[Figure]): The figures to attach to the test suite result.
         metric (Union[int, float]): Unit metrics attached to the test result.
         should_generate (bool): Whether to generate the description or not. Defaults to True.
+        instructions (Optional[str]): Instructions for the LLM to generate the description.
+        additional_context (Optional[str]): Additional context for the LLM to generate the description.
+        params (Optional[Dict[str, Any]]): Test parameters used to customize test behavior.
 
     Returns:
         str: The description to be logged with the test results.
     """
+    # Backwards compatibility: parameter instructions override environment variable
+    env_instructions = _get_llm_global_context()
+    # Parameter instructions take precedence and override environment variable
+    _instructions = instructions if instructions is not None else env_instructions
+
     # Check the feature flag first, then the environment variable
     llm_descriptions_enabled = (
         client_config.can_generate_llm_test_descriptions()
@@ -191,15 +272,7 @@ def get_result_description(
         not in ["0", "false"]
     )
 
-    # TODO: fix circular import
-    from validmind.ai.utils import is_configured
-
-    if (
-        should_generate
-        and (tables or figures)
-        and llm_descriptions_enabled
-        and is_configured()
-    ):
+    if should_generate and (tables or figures) and llm_descriptions_enabled:
         # get description future and set it as the description in the metadata
         # this will lazily retrieved so it can run in the background in parallel
         description = background_generate_description(
@@ -209,6 +282,9 @@ def get_result_description(
             figures=figures,
             metric=metric,
             title=title,
+            instructions=_instructions,
+            additional_context=additional_context,
+            params=params,
         )
 
     else:

{validmind-2.10.0rc2 → validmind-2.11.0}/validmind/ai/utils.py

@@ -35,13 +35,13 @@ class DescriptionFuture:
         self._future = future
 
     def get_description(self):
-        if isinstance(self._future, str):
+        if isinstance(self._future, tuple):
             description = self._future
         else:
             # This will block until the future is completed
             description = self._future.result()
 
-        return md_to_html(description, mathml=True)
+        return md_to_html(description[0], mathml=True), description[1]
 
 
 def get_client_and_model():

{validmind-2.10.0rc2 → validmind-2.11.0}/validmind/api_client.py

@@ -18,13 +18,12 @@ from urllib.parse import urlencode, urljoin
 import aiohttp
 import requests
 from aiohttp import FormData
-from ipywidgets import HTML, Accordion
 
 from .client_config import client_config
 from .errors import MissingAPICredentialsError, MissingModelIdError, raise_api_error
-from .logging import get_logger, init_sentry, log_api_operation, send_single_error
+from .logging import get_logger, log_api_operation
 from .utils import NumpyEncoder, is_html, md_to_html, run_async
-from .vm_models import Figure
+from .vm_models.figure import Figure
 
 logger = get_logger(__name__)
 
@@ -40,8 +39,6 @@ __api_session: Optional[aiohttp.ClientSession] = None
 @atexit.register
 def _close_session():
     """Closes the async client session at exit."""
-    global __api_session
-
     if __api_session and not __api_session.closed:
         try:
             loop = asyncio.get_event_loop()
@@ -86,6 +83,7 @@ def _get_session() -> aiohttp.ClientSession:
         __api_session = aiohttp.ClientSession(
             headers=_get_api_headers(),
             timeout=aiohttp.ClientTimeout(total=int(os.getenv("VM_API_TIMEOUT", 30))),
+            trust_env=True,
         )
 
     return __api_session
@@ -167,7 +165,7 @@ def _ping() -> Dict[str, Any]:
 
     client_info = r.json()
 
-    init_sentry(client_info.get("sentry_config", {}))
+    # Sentry removed: no telemetry initialization
 
     # Only show this confirmation the first time we connect to the API
     ack_connected = not client_config.model
@@ -245,14 +243,7 @@ def init(
 
 def reload():
     """Reconnect to the ValidMind API and reload the project configuration."""
-
-    try:
-        _ping()
-    except Exception as e:
-        # if the api host is https, assume we're not in dev mode and send to sentry
-        if _api_host.startswith("https://"):
-            send_single_error(e)
-        raise e
+    _ping()
 
 
 async def aget_metadata(content_id: str) -> Dict[str, Any]:
@@ -412,9 +403,7 @@ def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, A
     return run_async(alog_input, input_id, type, metadata)
 
 
-def log_text(
-    content_id: str, text: str, _json: Optional[Dict[str, Any]] = None
-) -> Dict[str, Any]:
+def log_text(content_id: str, text: str, _json: Optional[Dict[str, Any]] = None) -> str:
     """Logs free-form text to ValidMind API.
 
     Args:
@@ -427,7 +416,7 @@ def log_text(
         Exception: If the API call fails.
 
     Returns:
-        ipywidgets.Accordion: An accordion widget containing the logged text as HTML.
+        str: HTML string containing the logged text in an accordion format.
     """
     if not content_id or not isinstance(content_id, str):
         raise ValueError("`content_id` must be a non-empty string")
@@ -439,8 +428,10 @@ def log_text(
 
     log_text = run_async(alog_metadata, content_id, text, _json)
 
-    return Accordion(
-        children=[HTML(log_text["text"])],
+    from .vm_models.html_renderer import StatefulHTMLRenderer
+
+    return StatefulHTMLRenderer.render_accordion(
+        items=[log_text["text"]],
         titles=[f"Text Block: '{log_text['content_id']}'"],
     )
 
@@ -461,11 +452,11 @@ async def alog_metric(
     if value is None:
         raise ValueError("Must provide a value for the metric")
 
+    # Validate that value is a scalar (int or float)
     if not isinstance(value, (int, float)):
-        try:
-            value = float(value)
-        except (ValueError, TypeError):
-            raise ValueError("`value` must be a scalar (int or float)")
+        raise ValueError(
+            "Only scalar values (int or float) are allowed for logging metrics."
+        )
 
     if thresholds is not None and not isinstance(thresholds, dict):
         raise ValueError("`thresholds` must be a dictionary or None")
@@ -494,7 +485,7 @@ async def alog_metric(
 
 def log_metric(
     key: str,
-    value: float,
+    value: Union[int, float],
     inputs: Optional[List[str]] = None,
     params: Optional[Dict[str, Any]] = None,
     recorded_at: Optional[str] = None,
@@ -504,18 +495,19 @@ def log_metric(
     """Logs a unit metric.
 
     Unit metrics are key-value pairs where the key is the metric name and the value is
-    a scalar (int or float). These key-value pairs are associated with the currently
-    selected model (inventory model in the ValidMind Platform) and keys can be logged
-    to over time to create a history of the metric. On the ValidMind Platform, these metrics
-    will be used to create plots/visualizations for documentation and dashboards etc.
+    a scalar (int or float). These key-value pairs are associated
+    with the currently selected model (inventory model in the ValidMind Platform) and keys
+    can be logged to over time to create a history of the metric. On the ValidMind Platform,
+    these metrics will be used to create plots/visualizations for documentation and dashboards etc.
 
     Args:
         key (str): The metric key
-        value (Union[int, float]): The metric value
+        value (Union[int, float]): The metric value (scalar)
         inputs (List[str], optional): List of input IDs
         params (Dict[str, Any], optional): Parameters used to generate the metric
         recorded_at (str, optional): Timestamp when the metric was recorded
         thresholds (Dict[str, Any], optional): Thresholds for the metric
+        passed (bool, optional): Whether the metric passed validation thresholds
     """
     return run_async(
         alog_metric,

{validmind-2.10.0rc2 → validmind-2.11.0}/validmind/client.py

@@ -11,7 +11,11 @@ from typing import Any, Callable, Dict, List, Optional, Union
 import numpy as np
 import pandas as pd
 import polars as pl
-import torch
+
+try:
+    import torch  # noqa: F401
+except Exception:
+    torch = None  # type: ignore  # noqa: F401
 
 from .api_client import log_input as log_input
 from .client_config import client_config

validmind-2.11.0/validmind/datasets/llm/__init__.py

@@ -0,0 +1,14 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Entrypoint for LLM datasets.
+"""
+
+from .agent_dataset import LLMAgentDataset
+
+__all__ = [
+    "rag",
+    "LLMAgentDataset",
+]