validmind 2.1.1__tar.gz → 2.2.4__tar.gz

{validmind-2.1.1 → validmind-2.2.4}/PKG-INFO

@@ -1,14 +1,13 @@
 Metadata-Version: 2.1
 Name: validmind
-Version: 2.1.1
+Version: 2.2.4
 Summary: ValidMind Developer Framework
 License: Commercial License
 Author: Andres Rodriguez
 Author-email: andres@validmind.ai
-Requires-Python: >=3.8,<3.12
+Requires-Python: >=3.8.1,<3.12
 Classifier: License :: Other/Proprietary License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.8
 Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
@@ -26,6 +25,7 @@ Requires-Dist: evaluate (>=0.4.0,<0.5.0)
 Requires-Dist: ipywidgets (>=8.0.6,<9.0.0)
 Requires-Dist: kaleido (>=0.2.1,<0.3.0,!=0.2.1.post1)
 Requires-Dist: langdetect (>=1.0.9,<2.0.0)
+Requires-Dist: latex2mathml (>=3.77.0,<4.0.0)
 Requires-Dist: levenshtein (>=0.21.1,<0.22.0) ; extra == "all" or extra == "llm"
 Requires-Dist: llvmlite (>=0.42.0) ; python_version >= "3.12"
 Requires-Dist: llvmlite ; python_version >= "3.8" and python_full_version <= "3.11.0"
@@ -43,6 +43,7 @@ Requires-Dist: polars (>=0.20.15,<0.21.0)
 Requires-Dist: pycocoevalcap (>=1.2,<2.0) ; extra == "all" or extra == "llm"
 Requires-Dist: pypmml (>=0.9.17,<0.10.0)
 Requires-Dist: python-dotenv (>=0.20.0,<0.21.0)
+Requires-Dist: ragas (>=0.1.7,<0.2.0)
 Requires-Dist: rouge (>=1.0.1,<2.0.0)
 Requires-Dist: rpy2 (>=3.5.10,<4.0.0) ; extra == "all" or extra == "r-support"
 Requires-Dist: scikit-learn (>=1.0.2,<2.0.0)
@@ -55,6 +56,7 @@ Requires-Dist: sentry-sdk (>=1.24.0,<2.0.0)
 Requires-Dist: shap (>=0.42.0,<0.43.0)
 Requires-Dist: statsmodels (>=0.13.5,<0.14.0)
 Requires-Dist: tabulate (>=0.8.9,<0.9.0)
+Requires-Dist: textblob (>=0.18.0.post0,<0.19.0)
 Requires-Dist: textstat (>=0.7.3,<0.8.0)
 Requires-Dist: torch (>=1.10.0) ; extra == "all" or extra == "llm" or extra == "pytorch"
 Requires-Dist: torchmetrics (>=1.1.1,<2.0.0) ; extra == "all" or extra == "llm"

{validmind-2.1.1 → validmind-2.2.4}/pyproject.toml

@@ -10,9 +10,10 @@ description = "ValidMind Developer Framework"
 license = "Commercial License"
 name = "validmind"
 readme = "README.pypi.md"
-version = "2.1.1"
+version = "2.2.4"
 
 [tool.poetry.dependencies]
+python = ">=3.8.1,<3.12"
 aiohttp = {extras = ["speedups"], version = "^3.8.4"}
 arch = "^5.4.0"
 bert-score = "^0.3.13"
@@ -22,6 +23,7 @@ evaluate = "^0.4.0"
 ipywidgets = "^8.0.6"
 kaleido = "^0.2.1,!=0.2.1.post1"
 langdetect = "^1.0.9"
+latex2mathml = "^3.77.0"
 levenshtein = {version = "^0.21.1", optional = true}
 llvmlite = [
   {version = "*", python = ">=3.8,<=3.11"},
@@ -42,8 +44,8 @@ plotly-express = "^0.4.1"
 polars = "^0.20.15"
 pycocoevalcap = {version = "^1.2", optional = true}
 pypmml = "^0.9.17"
-python = ">=3.8,<3.12"
 python-dotenv = "^0.20.0"
+ragas = "^0.1.7"
 rouge = "^1.0.1"
 rpy2 = {version = "^3.5.10", optional = true}
 scikit-learn = "^1.0.2"
@@ -58,6 +60,7 @@ sentry-sdk = "^1.24.0"
 shap = "^0.42.0"
 statsmodels = "^0.13.5"
 tabulate = "^0.8.9"
+textblob = "^0.18.0.post0"
 textstat = "^0.7.3"
 torch = {version = ">=1.10.0", optional = true}
 torchmetrics = {version = "^1.1.1", optional = true}

validmind-2.2.4/validmind/__version__.py

@@ -0,0 +1 @@
+__version__ = "2.2.4"

{validmind-2.1.1 → validmind-2.2.4}/validmind/ai.py

@@ -8,45 +8,65 @@ import os
 from openai import AzureOpenAI, OpenAI
 
 SYSTEM_PROMPT = """
-You are an expert data scientist and MRM specialist tasked with providing concise and'
-objective insights based on the results of quantitative model or dataset analysis.
+You are an expert data scientist and MRM specialist.
+You are tasked with analyzing the results of a quantitative test run on some model or dataset.
+Your goal is to create a test description that will act as part of the model documentation.
+You will provide both the developer and other consumers of the documentation with a clear and concise "interpretation" of the results they will see.
+The overarching theme to maintain is MRM documentation.
 
-Examine the provided statistical test results and compose a brief summary. Highlight crucial
-insights, focusing on the distribution characteristics, central tendencies (such as mean or median),
-and the variability (including standard deviation and range) of the metrics. Evaluate how
-these statistics might influence the development and performance of a predictive model. Identify
-and explain any discernible trends or anomalies in the test results.
-
-Your analysis will act as the description of the result in the model documentation.
+Examine the provided statistical test results and compose a description of the results.
+This will act as the description and interpretation of the result in the model documentation.
+It will be displayed alongside the test results table and figures.
 
 Avoid long sentences and complex vocabulary.
 Structure the response clearly and logically.
-Use valid Markdown syntax to format the response (tables are supported).
+Use valid Markdown syntax to format the response.
+Respond only with your analysis and insights, not the verbatim test results.
+Respond only with the markdown content, no explanation or context for your response is necessary.
 Use the Test ID that is provided to form the Test Name e.g. "ClassImbalance" -> "Class Imbalance".
+
+Explain the test, its purpose, its mechanism/formula etc and why it is useful.
+If relevant, provide a very brief description of the way this test is used in model/dataset evaluation and how it is interpreted.
+Highlight the key insights from the test results. The key insights should be concise and easily understood.
+End the response with any closing remarks, summary or additional useful information.
+
 Use the following format for the response (feel free to modify slightly if necessary):
 ```
-**<Test Name>** <continue to explain what it does in detail>...
+**<Test Name>** calculates the xyz <continue to explain what it does in detail>...
+
+This test is useful for <explain why and for what this test is useful>...
 
-The results of this test <detailed explanation of the results>...
+**Key Insights:**
 
-In summary the following key insights can be gained:
+The following key insights can be identified in the test results:
 
-- **<key insight 1 - title>**: <explanation of key insight 1>
+- **<key insight 1 - title>**: <concise explanation of key insight 1>
 - ...<continue with any other key insights using the same format>
 ```
 It is very important that the text is nicely formatted and contains enough information to be useful to the user as documentation.
 """.strip()
+
+
 USER_PROMPT = """
-Test ID: {test_name}
-Test Description: {test_description}
-Test Results (the raw results of the test):
-{test_results}
-Test Summary (what the user sees in the documentation):
+Test ID: `{test_name}`
+
+<Test Docstring>
+{test_description}
+</Test Docstring>
+
+<Test Results Summary>
 {test_summary}
+</Test Results Summary>
 """.strip()
+
+
 USER_PROMPT_FIGURES = """
-Test ID: {test_name}
-Test Description: {test_description}
+Test ID: `{test_name}`
+
+<Test Docstring>
+{test_description}
+</Test Docstring>
+
 The attached plots show the results of the test.
 """.strip()
 
@@ -67,7 +87,7 @@ def __get_client_and_model():
 
     if "OPENAI_API_KEY" in os.environ:
         __client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
-        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4-turbo")
+        __model = os.environ.get("VM_OPENAI_MODEL", "gpt-4o")
 
     elif "AZURE_OPENAI_KEY" in os.environ:
         if "AZURE_OPENAI_ENDPOINT" not in os.environ:
@@ -113,22 +133,41 @@ class DescriptionFuture:
 def generate_description_async(
     test_name: str,
     test_description: str,
-    test_results: str,
     test_summary: str,
     figures: list = None,
 ):
     """Generate the description for the test results"""
-    client, _ = __get_client_and_model()
+    if not test_summary and not figures:
+        raise ValueError("No summary or figures provided - cannot generate description")
 
+    client, _ = __get_client_and_model()
     # get last part of test id
     test_name = test_name.split(".")[-1]
 
-    if not test_results and not test_summary:
-        if not figures:
-            raise ValueError("No results, summary or figures provided")
+    if test_summary:
+        return (
+            client.chat.completions.create(
+                model="gpt-4o",
+                messages=[
+                    {"role": "system", "content": SYSTEM_PROMPT},
+                    {
+                        "role": "user",
+                        "content": USER_PROMPT.format(
+                            test_name=test_name,
+                            test_description=test_description,
+                            test_summary=test_summary,
+                        ),
+                    },
+                ],
+            )
+            .choices[0]
+            .message.content.strip("```")
+            .strip()
+        )
 
-        response = client.chat.completions.create(
-            model="gpt-4-turbo",
+    return (
+        client.chat.completions.create(
+            model="gpt-4o",
             messages=[
                 {"role": "system", "content": SYSTEM_PROMPT},
                 {
@@ -154,30 +193,15 @@ def generate_description_async(
                 },
             ],
         )
-    else:
-        response = client.chat.completions.create(
-            model="gpt-4-turbo",
-            messages=[
-                {"role": "system", "content": SYSTEM_PROMPT},
-                {
-                    "role": "user",
-                    "content": USER_PROMPT.format(
-                        test_name=test_name,
-                        test_description=test_description,
-                        test_results=test_results,
-                        test_summary=test_summary,
-                    ),
-                },
-            ],
-        )
-
-    return response.choices[0].message.content.strip("```").strip()
+        .choices[0]
+        .message.content.strip("```")
+        .strip()
+    )
 
 
 def generate_description(
     test_name: str,
     test_description: str,
-    test_results: str,
     test_summary: str,
     figures: list = None,
 ):
@@ -185,7 +209,6 @@ def generate_description(
         generate_description_async,
         test_name,
         test_description,
-        test_results,
         test_summary,
         figures,
     )

{validmind-2.1.1 → validmind-2.2.4}/validmind/api_client.py

@@ -16,14 +16,13 @@ from io import BytesIO
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
 
 import aiohttp
-import mistune
 import requests
 from aiohttp import FormData
 
 from .client_config import client_config
 from .errors import MissingAPICredentialsError, MissingProjectIdError, raise_api_error
 from .logging import get_logger, init_sentry, send_single_error
-from .utils import NumpyEncoder, run_async
+from .utils import NumpyEncoder, md_to_html, run_async
 from .vm_models import Figure, MetricResult, ThresholdTestResults
 
 # TODO: can't import types from vm_models because of circular dependency
@@ -162,14 +161,20 @@ def __ping() -> Dict[str, Any]:
 
     init_sentry(client_info.get("sentry_config", {}))
 
+    # Only show this confirmation the first time we connect to the API
+    ack_connected = False
+    if client_config.project is None:
+        ack_connected = True
+
     client_config.project = client_info["project"]
     client_config.documentation_template = client_info.get("documentation_template", {})
     client_config.feature_flags = client_info.get("feature_flags", {})
 
-    logger.info(
-        f"Connected to ValidMind. Project: {client_config.project['name']}"
-        f" ({client_config.project['cuid']})"
-    )
+    if ack_connected:
+        logger.info(
+            f"Connected to ValidMind. Project: {client_config.project['name']}"
+            f" ({client_config.project['cuid']})"
+        )
 
 
 def reload():
@@ -344,7 +349,7 @@ async def log_metadata(
     """
     metadata_dict = {"content_id": content_id}
     if text is not None:
-        metadata_dict["text"] = mistune.html(text)
+        metadata_dict["text"] = md_to_html(text, mathml=True)
     if _json is not None:
         metadata_dict["json"] = _json
 
@@ -359,7 +364,11 @@ async def log_metadata(
 
 
 async def log_metrics(
-    metrics: List[MetricResult], inputs: List[str], output_template: str = None
+    metrics: List[MetricResult],
+    inputs: List[str],
+    output_template: str = None,
+    section_id: str = None,
+    position: int = None,
 ) -> Dict[str, Any]:
     """Logs metrics to ValidMind API.
 
@@ -367,6 +376,8 @@ async def log_metrics(
         metrics (list): A list of MetricResult objects
         inputs (list): A list of input keys (names) that were used to run the test
         output_template (str): The optional output template for the test
+        section_id (str): The section ID add a test driven block to the documentation
+        position (int): The position in the section to add the test driven block
 
     Raises:
         Exception: If the API call fails
@@ -374,7 +385,14 @@ async def log_metrics(
     Returns:
         dict: The response from the API
     """
+    params = {}
+    if section_id:
+        params["section_id"] = section_id
+    if position is not None:
+        params["position"] = position
+
     data = []
+
     for metric in metrics:
         metric_data = {
             **metric.serialize(),
@@ -389,6 +407,7 @@ async def log_metrics(
     try:
         return await _post(
             "log_metrics",
+            params=params,
             data=json.dumps(data, cls=NumpyEncoder, allow_nan=False),
         )
     except Exception as e:
@@ -397,7 +416,10 @@ async def log_metrics(
 
 
 async def log_test_result(
-    result: ThresholdTestResults, inputs: List[str], dataset_type: str = "training"
+    result: ThresholdTestResults,
+    inputs: List[str],
+    section_id: str = None,
+    position: int = None,
 ) -> Dict[str, Any]:
     """Logs test results information
 
@@ -407,8 +429,8 @@ async def log_test_result(
     Args:
         result (validmind.ThresholdTestResults): A ThresholdTestResults object
         inputs (list): A list of input keys (names) that were used to run the test
-        dataset_type (str, optional): The type of dataset. Can be one of
-            "training", "test", or "validation". Defaults to "training".
+        section_id (str, optional): The section ID add a test driven block to the documentation
+        position (int): The position in the section to add the test driven block
 
     Raises:
         Exception: If the API call fails
@@ -416,10 +438,16 @@ async def log_test_result(
     Returns:
         dict: The response from the API
     """
+    params = {}
+    if section_id:
+        params["section_id"] = section_id
+    if position is not None:
+        params["position"] = position
+
     try:
         return await _post(
             "log_test_results",
-            params={"dataset_type": dataset_type},
+            params=params,
             data=json.dumps(
                 {
                     **result.serialize(),
@@ -435,7 +463,7 @@ async def log_test_result(
 
 
 def log_test_results(
-    results: List[ThresholdTestResults], inputs, dataset_type: str = "training"
+    results: List[ThresholdTestResults], inputs
 ) -> List[Callable[..., Dict[str, Any]]]:
     """Logs test results information
 
@@ -445,8 +473,6 @@ def log_test_results(
     Args:
         results (list): A list of ThresholdTestResults objects
         inputs (list): A list of input keys (names) that were used to run the test
-        dataset_type (str, optional): The type of dataset. Can be one of "training",
-          "test", or "validation". Defaults to "training".
 
     Raises:
         Exception: If the API call fails
@@ -457,7 +483,7 @@ def log_test_results(
     try:
         responses = []  # TODO: use asyncio.gather
         for result in results:
-            responses.append(run_async(log_test_result, result, inputs, dataset_type))
+            responses.append(run_async(log_test_result, result, inputs))
     except Exception as e:
         logger.error("Error logging test results to ValidMind API")
         raise e

{validmind-2.1.1 → validmind-2.2.4}/validmind/client.py

@@ -21,20 +21,20 @@ from .errors import (
 )
 from .input_registry import input_registry
 from .logging import get_logger
+from .models.metadata import MetadataModel
 from .models.r_model import RModel
 from .template import get_template_test_suite
 from .template import preview_template as _preview_template
 from .test_suites import get_by_id as get_test_suite_by_id
 from .utils import get_dataset_info, get_model_info
 from .vm_models import TestInput, TestSuite, TestSuiteRunner
-from .vm_models.dataset import (
-    DataFrameDataset,
-    NumpyDataset,
-    PolarsDataset,
-    TorchDataset,
-    VMDataset,
+from .vm_models.dataset import DataFrameDataset, PolarsDataset, TorchDataset, VMDataset
+from .vm_models.model import (
+    ModelAttributes,
+    VMModel,
+    get_model_class,
+    is_model_metadata,
 )
-from .vm_models.model import VMModel, get_model_class
 
 pd.option_context("format.precision", 2)
 
@@ -129,7 +129,7 @@ def init_dataset(
         )
     elif dataset_class == "ndarray":
         logger.info("Numpy ndarray detected. Initializing VM Dataset instance...")
-        vm_dataset = NumpyDataset(
+        vm_dataset = VMDataset(
             input_id=input_id,
             raw_dataset=dataset,
             model=model,
@@ -175,8 +175,10 @@ def init_dataset(
 
 
 def init_model(
-    model: object,
-    input_id: str = None,
+    model: object = None,
+    input_id: str = "model",
+    attributes: dict = None,
+    predict_fn: callable = None,
     __log=True,
 ) -> VMModel:
     """
@@ -185,14 +187,13 @@ def init_model(
     also ensures we are creating a model supported libraries.
 
     Args:
-        model: A trained model
-        train_ds (vm.vm.Dataset): A training dataset (optional)
-        test_ds (vm.vm.Dataset): A testing dataset (optional)
-        validation_ds (vm.vm.Dataset): A validation dataset (optional)
+        model: A trained model or VMModel instance
         input_id (str): The input ID for the model (e.g. "my_model"). By default,
             this will be set to `model` but if you are passing this model as a
             test input using some other key than `model`, then you should set
             this to the same key.
+        attributes (dict): A dictionary of model attributes
+        predict_fn (callable): A function that takes an input and returns a prediction
 
     Raises:
         ValueError: If the model type is not supported
@@ -200,22 +201,64 @@ def init_model(
     Returns:
         vm.VMModel: A VM Model instance
     """
-    class_obj = get_model_class(model=model)
-    if not class_obj:
-        raise UnsupportedModelError(
-            f"Model type {class_obj} is not supported at the moment."
+    # vm_model = model if isinstance(model, VMModel) else None
+    # metadata = None
+
+    # if not vm_model:
+    #     class_obj = get_model_class(model=model, predict_fn=predict_fn)
+    #     if not class_obj:
+    #         if not attributes:
+    #             raise UnsupportedModelError(
+    #                 f"Model class {str(model.__class__)} is not supported at the moment."
+    #             )
+    #         elif not is_model_metadata(attributes):
+    #             raise UnsupportedModelError(
+    #                 f"Model attributes {str(attributes)} are missing required keys 'architecture' and 'language'."
+    #             )
+    vm_model = model if isinstance(model, VMModel) else None
+    class_obj = get_model_class(model=model, predict_fn=predict_fn)
+
+    if not vm_model and not class_obj:
+        if not attributes:
+            raise UnsupportedModelError(
+                f"Model class {str(model.__class__)} is not supported at the moment."
+            )
+
+        if not is_model_metadata(attributes):
+            raise UnsupportedModelError(
+                f"Model attributes {str(attributes)} are missing required keys 'architecture' and 'language'."
+            )
+
+    if isinstance(vm_model, VMModel):
+        vm_model.input_id = (
+            input_id if input_id != "model" else (vm_model.input_id or input_id)
         )
-    input_id = input_id or "model"
-    vm_model = class_obj(
-        input_id=input_id,
-        model=model,  # Trained model instance
-        attributes=None,
-    )
+        metadata = get_model_info(vm_model)
+    elif hasattr(class_obj, "__name__") and class_obj.__name__ == "PipelineModel":
+        vm_model = class_obj(
+            pipeline=model,
+            input_id=input_id,
+        )
+        # TODO: Add metadata for pipeline model
+        metadata = get_model_info(vm_model)
+    elif class_obj:
+        vm_model = class_obj(
+            input_id=input_id,
+            model=model,  # Trained model instance
+            predict_fn=predict_fn,
+        )
+        metadata = get_model_info(vm_model)
+    else:
+        vm_model = MetadataModel(
+            input_id=input_id, attributes=ModelAttributes.from_dict(attributes)
+        )
+        metadata = attributes
+
     if __log:
         log_input(
             name=input_id,
             type="model",
-            metadata=get_model_info(vm_model),
+            metadata=metadata,
         )
 
     input_registry.add(key=input_id, obj=vm_model)

validmind-2.2.4/validmind/datasets/llm/rag/__init__.py

@@ -0,0 +1,11 @@
+# Copyright © 2023-2024 ValidMind Inc. All rights reserved.
+# See the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""
+Entrypoint for classification datasets.
+"""
+
+__all__ = [
+    "rfp",
+]

validmind-2.2.4/validmind/datasets/llm/rag/datasets/rfp_existing_questions_client_1.csv

@@ -0,0 +1,30 @@
+Project_Title,RFP_Question_ID,question,ground_truth,Area,Last_Accessed_At,Requester,Status
+Gen AI-Driven Financial Advisory System,1,"What is your experience in developing AI-based applications, and can you provide examples of successful projects?","Our company has 15 years of experience in developing AI-based applications, with a strong portfolio in sectors such as healthcare, finance, and education. For instance, our project MediAI Insight for the healthcare industry demonstrated significant achievements in patient data analysis, resulting in a 30% reduction in diagnostic errors and a 40% improvement in treatment personalization. Our platform has engaged over 200 healthcare facilities, achieving a user satisfaction rate of 95%.",General,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,2,How do you ensure your AI-based apps remain up-to-date with the latest AI advancements and technologies?,"We maintain a dedicated R&D team focused on integrating the latest AI advancements into our applications. This includes regular updates and feature enhancements based on cutting-edge technologies such as GPT (Generative Pre-trained Transformer) for natural language understanding, CNNs (Convolutional Neural Networks) for advanced image recognition tasks, and DQN (Deep Q-Networks) for decision-making processes in complex environments. Our commitment to these AI methodologies ensures that our applications remain innovative, with capabilities that adapt to evolving market demands and client needs. This approach has enabled us to enhance the predictive accuracy of our financial forecasting tools by 25% and improve the efficiency of our educational content personalization by 40%",General,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,3,Can your AI-based applications be customized to meet specific user or business needs?,"Absolutely, customization is a core aspect of our offering. We work closely with clients to understand their specific needs and tailor our AI algorithms and app functionalities accordingly, using technologies such as TensorFlow for machine learning models, React for responsive UI/UX designs, and Kubernetes for scalable cloud deployment. This personalized approach allows us to optimize AI functionalities to match unique business processes, enhancing user experience and operational efficiency for each client. For example, for a retail client, we customized our recommendation engine to increase customer retention by 20% through more accurate and personalized product suggestions.",General,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,4,What measures do you take to ensure user privacy and data security in your AI-based apps?,"User privacy and data security are paramount. We implement robust measures such as end-to-end encryption to secure data transmissions, anonymization techniques to protect user identities, and comprehensive compliance with data protection laws like GDPR and CCPA. We also employ regular security audits and vulnerability assessments to ensure our systems are impenetrable. Additionally, our deployment of advanced intrusion detection systems and the use of secure coding practices reinforce our commitment to safeguarding user data at all times",General,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,5,How do you approach user interface and experience design in AI-based apps to ensure ease of use and engagement?,"Our design philosophy centers on simplicity and intuitiveness. We conduct extensive user research and testing to inform our UI/UX designs, ensuring that our AI-based apps are accessible and engaging for all users, regardless of their technical expertise. This includes applying principles from human-centered design, utilizing accessibility guidelines such as WCAG 2.1, and conducting iterative testing with diverse user groups. Our commitment to inclusivity and usability leads to higher user adoption rates and satisfaction. For instance, feedback-driven enhancements in our visual design have improved user engagement by over 30% across our applications.",General,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,6,Describe your support and maintenance services for AI-based applications post-launch.,"Post-launch, we offer comprehensive support and maintenance services, including regular updates, bug fixes, and performance optimization. Our support team is available 24/7 to assist with any issues or questions. We utilize a ticketing system that ensures swift response times, with an average initial response time of under 2 hours. Additionally, we provide monthly performance reports and hold quarterly reviews with clients to discuss system status and potential improvements. Our proactive approach includes using automated monitoring tools to detect and resolve issues before they impact users, ensuring that our applications perform optimally at all times. This service structure has been instrumental in maintaining a client satisfaction rate above 98%.",General,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,7,How do you measure the success and impact of your AI-based applications on client objectives?,"Success measurement is tailored to each project's objectives. We establish key performance indicators (KPIs) in collaboration with our clients, such as user engagement rates, efficiency improvements, or return on investment (ROI). We then regularly review these metrics using advanced analytics platforms and business intelligence tools to assess the app’s impact. Our approach includes monthly performance analysis meetings where we provide detailed reports and insights on metrics like session duration, user retention rates, and cost savings achieved through automation. We also implement A/B testing to continuously refine and optimize the application based on real-world usage data, ensuring that we make data-driven improvements that align closely with our clients' strategic goals.",General,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,8,"How do you ensure the ethical use of LLMs in your applications, particularly regarding bias mitigation and data privacy?","We adhere to ethical AI practices by implementing bias detection and mitigation techniques during the training of our Large Language Models (LLMs). This involves using diverse datasets to prevent skewed results and deploying algorithms specifically designed to identify and correct bias in AI outputs. For data privacy, we employ data anonymization and secure data handling protocols, ensuring compliance with GDPR, CCPA, and other relevant regulations. Our systems use state-of-the-art encryption methods for data at rest and in transit, and our data governance policies are rigorously audited by third-party security firms to maintain high standards of data integrity and confidentiality. This commitment extends to regular training for our staff on the latest privacy laws and ethical AI use to ensure that our practices are up-to-date and effective.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,9,"Can you describe the process of training your LLMs, including data sourcing, model selection, and validation methods?","Our LLM training process begins with the meticulous sourcing of diverse and comprehensive datasets from global sources, ensuring a rich variety that includes various languages, dialects, and cultural contexts. This diversity is critical for building models that perform equitably across different demographics. We leverage cutting-edge tools like Apache Kafka for real-time data streaming and Apache Hadoop for handling large datasets efficiently during preprocessing stages. For model architecture selection, we utilize TensorFlow and PyTorch frameworks to design and iterate on neural network structures that best suit each application's unique requirements, whether it's for predictive analytics in finance or customer service chatbots. Depending on the use case, we might choose from a variety of architectures such as Transformer models for their robust handling of sequential data or GANs (Generative Adversarial Networks) for generating new, synthetic data samples for training.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,10,How do you handle the continuous learning and updating of your LLMs to adapt to new data and evolving user needs?,"We implement advanced continuous learning mechanisms that allow our Large Language Models (LLMs) to adapt over time by incorporating new data and feedback loops, ensuring our models remain current and effective. We utilize incremental learning techniques where the model is periodically updated with fresh data without the need for retraining from scratch. This is facilitated by employing online learning algorithms such as Online Gradient Descent, which can quickly adjust model weights in response to new information.
+ To efficiently manage this continuous learning process, we use tools like Apache Spark for handling large-scale data processing in a distributed computing environment. This allows for seamless integration of new data streams into our training datasets. We also implement active learning cycles where the models request human feedback on specific outputs that are uncertain, thus refining model predictions over time based on actual user interactions and feedback.
+ Additionally, we incorporate reinforcement learning techniques where models are rewarded for improvements in performance metrics like accuracy and user engagement. This helps in fine-tuning the models' responses based on what is most effective in real-world scenarios.
+ For monitoring and managing these updates, we use TensorFlow Extended (TFX) for a robust end-to-end platform that ensures our models are consistently validated against performance benchmarks before being deployed. This continuous adaptation framework guarantees that our LLMs are not only keeping pace with evolving user needs and preferences but are also progressively enhancing their relevance and effectiveness.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,11,What measures do you take to ensure the transparency and explainability of decisions made by your LLMs?,"We prioritize transparency and explainability in our AI models by incorporating advanced features such as model interpretability layers and providing comprehensive documentation on how model decisions are made. This approach ensures that users can understand and trust the outputs of our Large Language Models (LLMs). To achieve this, we integrate tools like LIME (Local Interpretable Model-agnostic Explanations) and SHAP (SHapley Additive exPlanations) into our models. These tools allow us to break down and communicate the reasoning behind each model decision, fostering trust and facilitating easier audits by stakeholders.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,12,How do you assess and ensure the performance and scalability of your LLMs in high-demand scenarios?,"We conduct extensive performance testing under various load conditions to assess scalability and ensure our LLMs can handle high-demand scenarios efficiently. This involves using tools like Apache JMeter and LoadRunner to simulate different levels of user interaction and data volume, allowing us to evaluate how our systems perform under stress. Additionally, we employ scalable cloud infrastructure, utilizing services like Amazon Web Services (AWS) Elastic Compute Cloud (EC2) and Google Cloud Platform (GCP) Compute Engine, which support dynamic scaling. Optimization techniques such as auto-scaling groups and load balancers are implemented to ensure that our resources adjust automatically based on real-time demands, providing both robustness and cost efficiency.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,13,"Can you provide examples of successful deployments of your LLM-based applications, including the challenges faced and how they were addressed?","We can share case studies of successful LLM-based application deployments, highlighting specific challenges such as data scarcity or model interpretability, and detailing the strategies and solutions we implemented to overcome these challenges. For example, in a project involving natural language processing for a legal firm, we faced significant data scarcity. To address this, we employed techniques like synthetic data generation and transfer learning from related domains to enrich our training datasets. Additionally, the issue of model interpretability was critical for our client’s trust and regulatory compliance. We tackled this by integrating SHAP (SHapley Additive exPlanations) to provide clear, understandable insights into how our model's decisions were made, ensuring transparency and boosting user confidence in the AI system.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,14,What is your approach to integrating LLMs with existing systems and workflows within an organization?,"Our approach involves conducting a thorough analysis of the existing systems and workflows, designing integration plans that minimize disruption, and using APIs and custom connectors to ensure seamless integration of our LLM-based applications. We start by meticulously mapping the client's current infrastructure and operational flows to identify the most efficient points of integration. This is followed by the development of tailored integration plans that prioritize operational continuity and minimize downtime. To achieve seamless integration, we utilize robust APIs and develop custom connectors where necessary, ensuring compatibility with existing software platforms and databases. These tools allow for the smooth transfer of data and maintain the integrity and security of the system, ensuring that the new AI capabilities enhance functionality without compromising existing processes.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,15,"How do you plan to support and maintain LLM-based applications post-deployment, including handling model drift and providing updates?","Our post-deployment support is designed to ensure sustained performance and relevance of our LLM-based applications. We actively monitor for model drift to detect and address any degradation in model accuracy over time due to changes in underlying data patterns. This includes implementing automated systems that alert our team to potential drifts, allowing for timely interventions. Regular model updates and improvements are also part of our support protocol, ensuring that our solutions adapt to new data and evolving industry standards. Additionally, our dedicated technical support team is available to swiftly address any operational issues or adapt to changes in client requirements. This comprehensive support structure guarantees that our applications continue to deliver optimal performance and align with our clients' strategic objectives.",Large Language Models,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,16,How does your AI solution align with the NIST AI RMF's guidelines for trustworthy and responsible AI?,"Our AI solution is meticulously designed to align with the NIST AI Risk Management Framework (RMF) guidelines, ensuring adherence to principles of trustworthiness and responsibility. We have implemented comprehensive governance structures that oversee the ethical development and deployment of our AI technologies. This includes risk identification and assessment processes where potential risks are analyzed and categorized at each stage of the AI lifecycle. To manage these risks, we have instituted robust risk management controls that are deeply integrated into our development and operational processes. These controls are based on the NIST framework’s best practices, ensuring that our AI solutions are not only effective but also secure and ethical, maintaining transparency and accountability at all times.",AI Regulation,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,17,Can you describe the governance structures you have in place to manage AI risks as recommended by the NIST AI RMF?,"We have established an AI Risk Council that plays a pivotal role in overseeing AI risk management across our organization. This council is tasked with defining clear roles and responsibilities for AI governance, ensuring that there is a structured approach to managing AI risks. It also integrates AI risk management into our existing governance frameworks to enhance coherence and alignment with broader corporate policies and objectives. Additionally, the AI Risk Council promotes robust collaboration between various business units and our IT department. This collaboration is crucial for sharing insights, aligning strategies, and implementing comprehensive risk management practices effectively across the entire organization. This framework not only supports proactive risk management but also fosters an environment where AI technologies are used responsibly and ethically.",AI Regulation,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,18,How do you identify and assess AI risks in line with the NIST AI RMF's 'Map' function?,"We conduct thorough assessments of AI systems and the people using AI within our organization. This process involves meticulously identifying potential risks such as data privacy, security, bias, and legal compliance. We assess both the impact and the likelihood of each identified risk to effectively prioritize them. Our approach includes the use of sophisticated tools and methodologies, such as risk matrices and scenario analysis, to quantify and categorize risks accurately. This comprehensive assessment enables us to develop targeted risk mitigation strategies and allocate resources more efficiently, ensuring that the most critical risks are addressed promptly and effectively. This proactive risk management practice helps us maintain the integrity of our AI systems and uphold our ethical and legal responsibilities.",AI Regulation,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,19,"What measures do you take to ensure transparency and explainability in AI decision-making, as emphasized by the NIST AI RMF?","We prioritize transparency by incorporating explainability features into our AI models, providing detailed documentation on the decision-making processes, and ensuring that stakeholders can understand and trust the outputs of our AI systems. To achieve this, we integrate explainability tools like feature importance scores and decision trees that clearly outline how and why decisions are made by our AI. We supplement these technical tools with comprehensive documentation that describes the algorithms' functions in accessible language. This approach is designed to demystify the AI's operations for non-technical stakeholders, fostering a higher level of trust and acceptance. By ensuring that our AI systems are transparent and their workings understandable, we maintain open communication and build confidence among users and regulators alike.",AI Regulation,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,20,"How do you track and measure exposure to AI risks, and what metrics do you use, as suggested by the NIST AI RMF's 'Measure' function?","We have developed a set of Key Performance Indicators (KPIs) and metrics specifically designed to assess and analyze AI risk exposure across our systems. These metrics are tracked continuously to provide a clear, quantifiable measure of risk at any given time. To streamline this process, we utilize AI risk assessment tools that automate both data collection and analysis, enhancing the accuracy and efficiency of our monitoring efforts.
+ These tools employ advanced analytics to detect subtle shifts in risk patterns, enabling proactive risk management. Regular updates to our risk assessment protocols ensure that they remain aligned with current threat landscapes and regulatory requirements. This systematic monitoring and analysis not only help us maintain control over AI risks but also ensure that we can respond swiftly and effectively to any changes in risk levels, keeping our AI systems secure and compliant.",AI Regulation,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,21,Describe how your AI solutions manage and mitigate identified risks in accordance with the NIST AI RMF's 'Manage' function.,"We implement and maintain robust risk management controls to mitigate identified risks effectively. This comprehensive approach includes regular updates to our AI models to address evolving challenges and improve performance. We also implement stringent security measures, such as encryption, access controls, and continuous monitoring systems, to safeguard our data and systems from unauthorized access and potential breaches.
+ Furthermore, ensuring compliance with data protection laws is a critical part of our risk management strategy. We stay abreast of legal requirements in all operational jurisdictions, such as GDPR in Europe and CCPA in California, and integrate compliance measures into our AI deployments. Regular audits, both internal and by third-party assessors, help ensure that our practices are up-to-date and that we maintain the highest standards of data privacy and security. This holistic approach to risk management enables us to maintain trust and reliability in our AI applications.",AI Regulation,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,22,How do you ensure that your AI solutions are compliant with U.S. regulations on data privacy and security?,"We ensure compliance with U.S. regulations such as the Federal Information Security Modernization Act (FISMA) and other applicable laws and directives by adopting a risk-based approach to control selection and specification. This approach meticulously considers the constraints and requirements imposed by these regulations. We conduct regular audits and assessments to verify that our security controls meet or exceed the stipulated standards, ensuring that all our data handling and processing activities are fully compliant.
+ Our compliance framework is designed to adapt to the specific needs of the environments in which our systems operate, integrating best practices and guidance from regulatory bodies. We also engage with legal and compliance experts to stay updated on any changes in legislation, ensuring our practices remain in line with the latest requirements. This proactive and informed approach allows us to manage risk effectively while maintaining the highest levels of data protection and security as mandated by U.S. law.",AI Regulation,18/12/2023,Bank A,Under Review
+Gen AI-Driven Financial Advisory System,23,"In what ways do you contribute to the continual improvement of AI risk management practices, as envisioned by the NIST AI RMF?","We actively participate in industry working groups and public-private partnerships to contribute to the continual improvement of AI risk management practices. Our engagement in these collaborative efforts not only allows us to share our insights and strategies but also enables us to learn from the collective experiences of the industry, helping to elevate the standards of AI safety and reliability across the board. Additionally, we stay abreast of updates to the NIST AI Risk Management Framework (RMF) and adjust our practices accordingly. This commitment to staying current ensures that our risk management approaches align with the latest guidelines and best practices, reinforcing our dedication to leading-edge, responsible AI development and deployment.",AI Regulation,18/12/2023,Bank A,Under Review