validmind 2.12.4__tar.gz → 2.13.0__tar.gz

{validmind-2.12.4 → validmind-2.13.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: validmind
-Version: 2.12.4
+Version: 2.13.0
 Summary: ValidMind Library
 License: DUAL LICENSE NOTICE
          

{validmind-2.12.4 → validmind-2.13.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "validmind"
-version = "2.12.4"
+version = "2.13.0"
 description = "ValidMind Library"
 readme = "README.pypi.md"
 requires-python = ">=3.9,<3.15"

{validmind-2.12.4 → validmind-2.13.0}/validmind/__init__.py

@@ -52,6 +52,8 @@ from . import scorers as scorer
 from .__version__ import __version__  # noqa: E402
 from .api_client import init, log_metric, log_test_result, log_text, reload
 from .client import (  # noqa: E402
+    generate_documentation_text,
+    get_content_ids,
     get_test_suite,
     init_dataset,
     init_model,
@@ -59,6 +61,7 @@ from .client import (  # noqa: E402
     preview_template,
     run_documentation_tests,
     run_test_suite,
+    run_text_generation,
 )
 from .experimental import agents as experimental_agent
 from .tests.decorator import scorer as scorer_decorator
@@ -116,11 +119,14 @@ __all__ = [  # noqa
     "init_dataset",
     "init_model",
     "init_r_model",
+    "generate_documentation_text",
     "get_test_suite",
+    "get_content_ids",
     "log_metric",
     "preview_template",
     "print_env",
     "reload",
+    "run_text_generation",
     "run_documentation_tests",
     # log metric function (for direct/bulk/retroactive logging of metrics)
     # test suite functions (less common)

validmind-2.13.0/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.13.0"

{validmind-2.12.4 → validmind-2.13.0}/validmind/api_client.py

@@ -278,6 +278,7 @@ async def alog_metadata(
     content_id: str,
     text: Optional[str] = None,
     _json: Optional[Dict[str, Any]] = None,
+    section_id: Optional[str] = None,
 ) -> Dict[str, Any]:
     """Logs free-form metadata to ValidMind API.
 
@@ -285,6 +286,8 @@ async def alog_metadata(
         content_id (str): Unique content identifier for the metadata.
         text (str, optional): Free-form text to assign to the metadata. Defaults to None.
         _json (dict, optional): Free-form key-value pairs to assign to the metadata. Defaults to None.
+        section_id (str, optional): Section ID to append the text block to when the
+            content ID does not already exist.
 
     Raises:
         Exception: If the API call fails.
@@ -298,9 +301,14 @@ async def alog_metadata(
     if _json is not None:
         metadata_dict["json"] = _json
 
+    request_params = {}
+    if section_id:
+        request_params["section_id"] = section_id
+
     try:
         return await _post(
             "log_metadata",
+            params=request_params,
             data=json.dumps(metadata_dict, cls=NumpyEncoder, allow_nan=False),
         )
     except Exception as e:
@@ -445,37 +453,196 @@ def log_input(input_id: str, type: str, metadata: Dict[str, Any]) -> Dict[str, A
     return run_async(alog_input, input_id, type, metadata)
 
 
-def log_text(content_id: str, text: str, _json: Optional[Dict[str, Any]] = None) -> str:
-    """Logs free-form text to ValidMind API.
+def _validate_log_text_context(
+    context: Optional[Dict[str, Any]],
+) -> Optional[Dict[str, List[str]]]:
+    """Validate supported AI generation context for ``log_text``."""
+    if context is None:
+        return None
+
+    if not isinstance(context, dict):
+        raise ValueError("`context` must be a dictionary or None")
+
+    allowed_keys = {"content_ids"}
+    unknown_keys = set(context.keys()) - allowed_keys
+    if unknown_keys:
+        raise ValueError(
+            "Unsupported `context` keys: "
+            f"{', '.join(sorted(unknown_keys))}. Only `content_ids` is supported."
+        )
+
+    content_ids = context.get("content_ids")
+    if content_ids is None:
+        raise ValueError("`context` must include `content_ids` when provided")
+    if not isinstance(content_ids, list) or not content_ids:
+        raise ValueError("`context['content_ids']` must be a non-empty list")
+    if any(
+        not isinstance(content_id, str) or not content_id for content_id in content_ids
+    ):
+        raise ValueError("`context['content_ids']` must contain only non-empty strings")
+
+    return {"content_ids": content_ids}
+
+
+def generate_qualitative_text(text_generation_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Generate qualitative text using the ValidMind AI API."""
+    r = requests.post(
+        url=_get_url("ai/generate/qualitative_text_generation"),
+        headers=_get_api_headers(),
+        json=text_generation_data,
+    )
+
+    if r.status_code != 200:
+        raise_api_error(r.text)
+
+    return r.json()
+
+
+def _normalize_logged_text(text: str, field_name: str) -> str:
+    """Validate text content and convert Markdown to HTML when needed."""
+    if not isinstance(text, str) or not text:
+        raise ValueError(f"`{field_name}` must be a non-empty string")
+
+    if not is_html(text):
+        return md_to_html(text, mathml=True)
+
+    return text
+
+
+def _validate_manual_log_text_args(
+    text: str, prompt: Optional[str], context: Optional[Dict[str, Any]]
+) -> str:
+    """Validate manual log_text arguments."""
+    if prompt is not None:
+        raise ValueError("`prompt` is only supported when `text` is omitted")
+    if context is not None:
+        raise ValueError("`context` is only supported when `text` is omitted")
+
+    return _normalize_logged_text(text, "text")
+
+
+def _build_log_text_generation_request(
+    content_id: str,
+    prompt: Optional[str],
+    context: Optional[Dict[str, Any]],
+    section_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Build the request payload for AI-assisted text generation."""
+    request_data = {
+        "content_id": content_id,
+        "generate": True,
+    }
+
+    if prompt is not None:
+        if not isinstance(prompt, str) or not prompt:
+            raise ValueError("`prompt` must be a non-empty string")
+        request_data["prompt"] = prompt
+
+    if section_id is not None:
+        if not isinstance(section_id, str) or not section_id:
+            raise ValueError("`section_id` must be a non-empty string")
+        request_data["section_id"] = section_id
+
+    validated_context = _validate_log_text_context(context)
+    if validated_context is not None:
+        request_data["context"] = validated_context
+
+    return request_data
+
+
+def _generate_log_text(
+    content_id: str,
+    prompt: Optional[str],
+    context: Optional[Dict[str, Any]],
+    section_id: Optional[str] = None,
+) -> str:
+    """Generate text for log_text and normalize it to HTML."""
+    request_data = _build_log_text_generation_request(
+        content_id,
+        prompt,
+        context,
+        section_id=section_id,
+    )
+    generated_text = generate_qualitative_text(request_data)["content"]
+    return _normalize_logged_text(generated_text, "generated text")
+
+
+def _render_logged_text(logged_text: Dict[str, Any]) -> str:
+    """Render logged text as notebook-friendly HTML."""
+    from .vm_models.html_renderer import StatefulHTMLRenderer
+
+    return StatefulHTMLRenderer.render_accordion(
+        items=[logged_text["text"]],
+        titles=[f"Text Block: '{logged_text['content_id']}'"],
+    )
+
+
+async def alog_text(
+    content_id: str,
+    text: Optional[str] = None,
+    prompt: Optional[str] = None,
+    context: Optional[Dict[str, Any]] = None,
+    _json: Optional[Dict[str, Any]] = None,
+    section_id: Optional[str] = None,
+) -> Dict[str, Any]:
+    """Async variant of ``log_text`` that logs or generates text."""
+    if not content_id or not isinstance(content_id, str):
+        raise ValueError("`content_id` must be a non-empty string")
+
+    if text is not None:
+        text = _validate_manual_log_text_args(text, prompt, context)
+    else:
+        text = _generate_log_text(
+            content_id,
+            prompt,
+            context,
+            section_id=section_id,
+        )
+
+    return await alog_metadata(content_id, text, _json, section_id=section_id)
+
+
+def log_text(
+    content_id: str,
+    text: Optional[str] = None,
+    prompt: Optional[str] = None,
+    context: Optional[Dict[str, Any]] = None,
+    _json: Optional[Dict[str, Any]] = None,
+    section_id: Optional[str] = None,
+) -> str:
+    """Logs or generates free-form text to ValidMind API.
 
     Args:
         content_id (str): Unique content identifier for the text.
-        text (str): The text to log. Will be converted to HTML with MathML support.
+        text (str, optional): The text to log. Will be converted to HTML with
+            MathML support when Markdown is provided. If omitted, text is
+            generated using the qualitative text generation backend.
+        prompt (str, optional): Custom prompt used for AI-assisted text
+            generation. Only supported when `text` is omitted.
+        context (dict, optional): Context object for AI-assisted text
+            generation. When omitted, the full document is used as context.
+            Currently only supports `{"content_ids": [<content_id>, ...]}`.
         _json (dict, optional): Additional metadata to associate with the text. Defaults to None.
+        section_id (str, optional): Section ID to append the text block to when the
+            content ID does not already exist.
 
     Raises:
-        ValueError: If content_id or text are empty or not strings.
+        ValueError: If arguments are invalid or use incompatible combinations.
         Exception: If the API call fails.
 
     Returns:
         str: HTML string containing the logged text in an accordion format.
     """
-    if not content_id or not isinstance(content_id, str):
-        raise ValueError("`content_id` must be a non-empty string")
-    if not text or not isinstance(text, str):
-        raise ValueError("`text` must be a non-empty string")
-
-    if not is_html(text):
-        text = md_to_html(text, mathml=True)
-
-    log_text = run_async(alog_metadata, content_id, text, _json)
-
-    from .vm_models.html_renderer import StatefulHTMLRenderer
-
-    return StatefulHTMLRenderer.render_accordion(
-        items=[log_text["text"]],
-        titles=[f"Text Block: '{log_text['content_id']}'"],
+    logged_text = run_async(
+        alog_text,
+        content_id=content_id,
+        text=text,
+        prompt=prompt,
+        context=context,
+        _json=_json,
+        section_id=section_id,
     )
+    return _render_logged_text(logged_text)
 
 
 async def alog_metric(

{validmind-2.12.4 → validmind-2.13.0}/validmind/client.py

@@ -6,6 +6,7 @@
 Client interface for all data and model validation functions
 """
 
+import time
 from typing import Any, Callable, Dict, List, Optional, Union
 
 import numpy as np
@@ -17,6 +18,7 @@ try:
 except Exception:
     torch = None  # type: ignore  # noqa: F401
 
+from . import api_client
 from .api_client import log_input as log_input
 from .client_config import client_config
 from .errors import (
@@ -31,18 +33,23 @@ from .input_registry import input_registry
 from .logging import get_logger
 from .models.metadata import MetadataModel
 from .models.r_model import RModel
-from .template import get_template_test_suite
+from .template import get_template_content_ids, get_template_test_suite
 from .template import preview_template as _preview_template
 from .test_suites import get_by_id as get_test_suite_by_id
+from .tests.run import _get_run_metadata
+from .utils import display as vm_display
 from .utils import get_dataset_info, get_model_info
 from .vm_models import TestSuite, TestSuiteRunner
 from .vm_models.dataset import DataFrameDataset, PolarsDataset, TorchDataset, VMDataset
+from .vm_models.html_progress import HTMLLabel, HTMLProgressBar
 from .vm_models.model import (
     ModelAttributes,
     VMModel,
     get_model_class,
     is_model_metadata,
 )
+from .vm_models.result import TextGenerationResult
+from .vm_models.text_generation_summary import DocumentationTextSummary
 
 pd.option_context("format.precision", 2)
 
@@ -433,6 +440,145 @@ def preview_template() -> None:
     _preview_template(client_config.documentation_template)
 
 
+def get_content_ids(section_ids: Optional[Union[str, List[str]]] = None) -> List[str]:
+    """Get content IDs for one or more documentation template sections.
+
+    Args:
+        section_ids: Section ID or list of section IDs. If omitted, all content
+            IDs in the current documentation template are returned.
+
+    Returns:
+        A list of content IDs in template order.
+    """
+    if client_config.documentation_template is None:
+        raise MissingDocumentationTemplate(
+            "No documentation template found. Please run `vm.init()`"
+        )
+
+    return get_template_content_ids(client_config.documentation_template, section_ids)
+
+
+def run_text_generation(
+    content_id: str,
+    prompt: Optional[str] = None,
+    section_id: Optional[str] = None,
+    context: Optional[Dict[str, Any]] = None,
+    show: bool = True,
+) -> TextGenerationResult:
+    """Generate qualitative text and return a loggable result object.
+
+    Args:
+        content_id: Content ID to generate text for.
+        prompt: Custom prompt for text generation.
+        section_id: Optional section ID used to guide generation for new content
+            and later placement during logging.
+        context: Optional context object for text generation.
+        show: Whether to display the generated result.
+
+    Returns:
+        A TextGenerationResult containing the generated text.
+    """
+    start_time = time.perf_counter()
+    description = api_client._generate_log_text(
+        content_id, prompt, context, section_id=section_id
+    )
+    metadata = _get_run_metadata(duration_seconds=time.perf_counter() - start_time)
+    result = TextGenerationResult(
+        result_id=content_id,
+        result_type="qualitative_text_generation",
+        content_id=content_id,
+        title=f"Text Generation: {content_id}",
+        description=description,
+        metadata=metadata,
+        prompt=prompt,
+        section_id=section_id,
+        context=context,
+        _was_description_generated=True,
+    )
+
+    if show:
+        result.show()
+
+    return result
+
+
+def generate_documentation_text(
+    config: Dict[str, Dict[str, Any]],
+) -> Dict[str, TextGenerationResult]:
+    """Generate and log document text blocks from a configuration dictionary.
+
+    The config is keyed by target content ID. Each value may contain:
+        - `section_id` (optional): Section to append the content block to when it
+          does not already exist in the template.
+        - `prompt` (optional): Prompt override for generation.
+        - `context` (optional): Context object passed to `run_text_generation()`,
+          e.g. `{"content_ids": [...]}`.
+
+    Args:
+        config: Mapping of content IDs to generation settings.
+
+    Returns:
+        A dictionary mapping content IDs to generated and logged results.
+    """
+    results = {}
+    num_tasks = len(config) * 2
+    html_pbar = None
+    html_pbar_description = None
+    template = client_config.documentation_template or {}
+
+    if num_tasks:
+        html_pbar_description = HTMLLabel(value="Generating documentation text...")
+        html_pbar = HTMLProgressBar(
+            max_value=num_tasks,
+            description="Generating documentation text...",
+        )
+        html_pbar.display()
+
+    def _update_progress_message(message: str) -> None:
+        if html_pbar:
+            html_pbar.update(html_pbar.value, message)
+        if html_pbar_description:
+            html_pbar_description.update(message)
+
+    def _increment_progress() -> None:
+        if html_pbar:
+            html_pbar.update(html_pbar.value + 1)
+
+    try:
+        for content_id, spec in config.items():
+            progress_message = f"Sending result to ValidMind: {content_id}..."
+            _update_progress_message(progress_message)
+            result = run_text_generation(
+                content_id=content_id,
+                prompt=spec.get("prompt"),
+                section_id=spec.get("section_id"),
+                context=spec.get("context"),
+                show=False,
+            )
+            _increment_progress()
+
+            result.log()
+            _increment_progress()
+            results[content_id] = result
+    finally:
+        if html_pbar:
+            html_pbar.update(num_tasks, "Documentation text generation complete!")
+            html_pbar.close()
+        if html_pbar_description:
+            html_pbar_description.update("Documentation text generation complete!")
+
+    if results:
+        summary = DocumentationTextSummary(
+            title=template.get("template_name", "Documentation Text"),
+            description=template.get("description", ""),
+            results=results,
+            template_sections=template.get("sections"),
+        )
+        vm_display(summary)
+
+    return results
+
+
 def run_documentation_tests(
     section: Optional[str] = None,
     send: bool = True,

validmind-2.13.0/validmind/datasets/classification/config.json

@@ -0,0 +1,121 @@
+{
+  "model_overview": {
+    "section_id": "model_overview",
+    "context": {
+      "content_ids": [
+        "validmind.data_validation.DatasetDescription",
+        "validmind.model_validation.sklearn.ClassifierPerformance:in_sample",
+        "validmind.model_validation.sklearn.ClassifierPerformance:out_of_sample"
+      ]
+    }
+  },
+  "intended_use": {
+    "section_id": "intended_use"
+  },
+  "regulatory_requirements": {
+    "section_id": "regulatory_requirements",
+    "prompt": "Write a comprehensive paragraphs describing the regulatory requirements for this model, assuming SR 11-7 is the primary regulatory framework."
+  },
+  "model_limitations": {
+    "section_id": "model_limitations",
+    "context": {
+      "content_ids": [
+        "validmind.model_validation.sklearn.TrainingTestDegradation",
+        "validmind.model_validation.sklearn.MinimumAccuracy",
+        "validmind.model_validation.sklearn.MinimumF1Score",
+        "validmind.model_validation.sklearn.MinimumROCAUCScore"
+      ]
+    }
+  },
+  "model_selection": {
+    "section_id": "model_selection",
+    "context": {
+      "content_ids": [
+        "validmind.model_validation.ModelMetadata",
+        "validmind.model_validation.sklearn.ClassifierPerformance:in_sample",
+        "validmind.model_validation.sklearn.ClassifierPerformance:out_of_sample"
+      ]
+    }
+  },
+  "dataset_summary_text": {
+    "context": {
+      "content_ids": [
+        "validmind.data_validation.DatasetDescription"
+      ]
+    }
+  },
+  "data_quality_tests_text": {
+    "context": {
+      "content_ids": [
+        "validmind.data_validation.ClassImbalance",
+        "validmind.data_validation.Duplicates",
+        "validmind.data_validation.HighCardinality",
+        "validmind.data_validation.MissingValues",
+        "validmind.data_validation.Skewness",
+        "validmind.data_validation.UniqueRows",
+        "validmind.data_validation.TooManyZeroValues",
+        "validmind.data_validation.IQROutliersTable",
+        "validmind.data_validation.IQROutliersBarPlot"
+      ]
+    }
+  },
+  "feature_selection": {
+    "context": {
+      "content_ids": [
+        "validmind.data_validation.PearsonCorrelationMatrix",
+        "validmind.data_validation.HighPearsonCorrelation"
+      ]
+    }
+  },
+  "model_validation_tests_text": {
+    "context": {
+      "content_ids": [
+        "validmind.model_validation.sklearn.ConfusionMatrix",
+        "validmind.model_validation.sklearn.ClassifierPerformance:in_sample",
+        "validmind.model_validation.sklearn.ClassifierPerformance:out_of_sample",
+        "validmind.model_validation.sklearn.PrecisionRecallCurve",
+        "validmind.model_validation.sklearn.ROCCurve",
+        "validmind.model_validation.sklearn.MinimumAccuracy",
+        "validmind.model_validation.sklearn.MinimumF1Score",
+        "validmind.model_validation.sklearn.MinimumROCAUCScore"
+      ]
+    }
+  },
+  "validmind.model_validation.sklearn.SHAPGlobalImportance_global_importance_text": {
+    "context": {
+      "content_ids": [
+        "validmind.model_validation.sklearn.SHAPGlobalImportance"
+      ]
+    }
+  },
+  "model_weak_spots_description": {
+    "context": {
+      "content_ids": [
+        "validmind.model_validation.sklearn.WeakspotsDiagnosis"
+      ]
+    }
+  },
+  "model_overfit_regions_description": {
+    "context": {
+      "content_ids": [
+        "validmind.model_validation.sklearn.OverfitDiagnosis"
+      ]
+    }
+  },
+  "model_robustness_description": {
+    "context": {
+      "content_ids": [
+        "validmind.model_validation.sklearn.RobustnessDiagnosis"
+      ]
+    }
+  },
+  "monitoring_plan": {
+    "prompt": "Write two comprehensive paragraphs for the monitoring plan section."
+  },
+  "monitoring_implementation": {
+    "prompt": "Write two comprehensive paragraphs for the monitoring implementation section."
+  },
+  "governance_plan": {
+    "prompt": "Write two comprehensive paragraphs for the governance plan section."
+  }
+}

{validmind-2.12.4 → validmind-2.13.0}/validmind/datasets/classification/customer_churn.py

@@ -2,7 +2,9 @@
 # Refer to the LICENSE file in the root of this repository for details.
 # SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
 
+import json
 import os
+from typing import Any, Dict
 
 import pandas as pd
 from sklearn.model_selection import train_test_split
@@ -17,6 +19,7 @@ from . import (
 
 current_path = os.path.dirname(os.path.abspath(__file__))
 dataset_path = os.path.join(current_path, "datasets")
+config_path = os.path.join(current_path, "config.json")
 
 drop_columns = ["RowNumber", "CustomerId", "Surname"]
 boolean_columns = ["Gender"]
@@ -60,6 +63,35 @@ def preprocess(df):
     return train_df, validation_df, test_df
 
 
+def get_demo_text_config() -> Dict[str, Dict[str, Any]]:
+    """Return text generation config for the customer churn demo.
+
+    The config is keyed by target `content_id`. Each entry describes how
+    `vm.run_text_generation()` should populate that block.
+
+    Supported fields per content ID:
+        - `section_id` (optional): Only needed when the content block does not
+          already exist in the template and should be appended to a section.
+        - `prompt` (optional): Per-block prompt override for generation.
+        - `context` (optional): Context object passed through to
+          `vm.run_text_generation()`. When present, it follows the same shape
+          expected by the library, e.g. `{"content_ids": [...]}`.
+
+    Returns:
+        Dict[str, Dict[str, Any]]: A previewable text-generation configuration
+        dictionary for the customer churn documentation demo.
+    """
+    with open(config_path, encoding="utf-8") as config_file:
+        config = json.load(config_file)
+
+    for _, spec in config.items():
+        context = spec.get("context")
+        if context is not None:
+            context.setdefault("content_ids", [])
+
+    return config
+
+
 def get_demo_test_config(test_suite=None):
     """
     Returns input configuration for the default documentation