aiverify-moonshot 0.4.8__tar.gz → 0.4.10__tar.gz

aiverify_moonshot-0.4.10/.github/workflows/smoke-test-cli.yaml

@@ -0,0 +1,136 @@
+# Cli Smoke Test
+
+name: Moonshot CLI Smoke Test
+
+on:
+  # Runs on Pull Request Review
+  pull_request_review:
+   types: [submitted]
+   branches:
+     - 'dev_main'
+
+  # Run this workflow manually from Actions tab
+  workflow_dispatch:
+    inputs:
+      moonshot_branch:
+        description: 'Moonshot Branch / Tag Name'
+        required: true
+        default: 'dev_main'
+        type: string
+      moonshot_data_branch:
+        description: 'Moonshot Data Branch / Tag Name'
+        required: true
+        default: 'dev_main'
+        type: string
+      moonshot_ui_branch:
+        description: 'Moonshot UI Branch / Tag Name'
+        required: true
+        default: 'dev_main'
+        type: string
+
+# Allow one concurrent deployment
+concurrency:
+  group: ${{ github.repository }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  smoke-test:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+
+    - name: Checkout Moonshot (Pull Request Review)
+      if: github.event_name == 'pull_request_review'
+      uses: actions/checkout@v4
+      with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
+
+    - name: Checkout Moonshot (Workflow Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      uses: actions/checkout@v4
+      with:
+          repository: aiverify-foundation/moonshot
+          ref: ${{ inputs.moonshot_branch }}
+    
+    - name: Setup Python 3.11
+      uses: actions/setup-python@v4
+      with:
+          python-version: '3.11'
+    
+    - name: Setup Moonshot
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+        pip install -r requirements.txt
+        touch .env
+        echo "${{ secrets.ENVIRONMENT_VARS }}" >> .env
+
+    - name: Checkout Moonshot Data (Pull Request Review)
+      if: github.event_name == 'pull_request_review'
+      uses: actions/checkout@v4
+      with: 
+         repository: aiverify-foundation/moonshot-data
+         ref: ${{ vars.MOONSHOT_DATA_BRANCH }}
+         path: moonshot-data
+
+    - name: Checkout Moonshot Data (Workflow Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      uses: actions/checkout@v4
+      with: 
+         repository: aiverify-foundation/moonshot-data
+         ref: ${{ inputs.moonshot_data_branch }}
+         path: moonshot-data
+
+    - name: Setup Moonshot Data
+      run: |
+        source venv/bin/activate
+        cd moonshot-data
+        pip install -r requirements.txt
+
+    - name: Checkout Moonshot UI (Pull Request Review)
+      if: github.event_name == 'pull_request_review'
+      uses: actions/checkout@v4
+      with:
+        repository: aiverify-foundation/moonshot-ui
+        ref: ${{ vars.MOONSHOT_UI_BRANCH }}
+        path: moonshot-ui
+
+    - name: Checkout Moonshot UI (Workflow Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      uses: actions/checkout@v4
+      with:
+        repository: aiverify-foundation/moonshot-ui
+        ref: ${{ inputs.moonshot_ui_branch }}
+        path: moonshot-ui
+    
+    - name: Setup Moonshot UI
+      run: |
+        cd moonshot-ui
+        npm ci
+        npm run build
+        cd ../
+
+    - name: Checkout Smoke Test 
+      uses: actions/checkout@v4
+      with:
+          repository: aiverify-foundation/moonshot-smoke-testing
+          path: moonshot-smoke-testing
+
+    - name: Run Smoke Test
+      env: 
+        AZURE_OPENAI_URI: ${{ secrets.AZURE_OPENAI_URI }}
+        AZURE_OPENAI_TOKEN: ${{ secrets.AZURE_OPENAI_TOKEN }}
+        ADDITIONAL_PARAMETERS: ${{ secrets.ADDITIONAL_PARAMETERS }}
+        MOONSHOT_URL: ${{ secrets.MOONSHOT_URL }}
+        MOONSHOT_PORT_NUMBER: ${{ secrets.MOONSHOT_PORT_NUMBER }}
+        CLI_DIR: ${{ secrets.CLI_DIR }}
+      run: |
+         source venv/bin/activate
+         cd moonshot-smoke-testing
+         cd tests
+         pip install python-dotenv
+         pip install pytest
+         pytest test_cli.py

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/.github/workflows/smoke-test.yaml

@@ -3,7 +3,7 @@
 name: Moonshot Smoke Test
 
 on:
-  # Runs on Pull Request Review
+  # Runs on Pull Request Review 
   pull_request_review:
    types: [submitted]
    branches:
@@ -35,9 +35,9 @@ concurrency:
 
 jobs:
   smoke-test:
-
+    if: (github.event_name == 'pull_request_review' && github.event.pull_request.base.ref == 'dev_main') || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 60
 
     steps:
 

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/LICENSE.md

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2024 The Moonshot Team
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: aiverify-moonshot
-Version: 0.4.8
+Version: 0.4.10
 Summary: AI Verify advances Gen AI testing with Project Moonshot.
 Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
 Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
@@ -46,7 +46,7 @@ Description-Content-Type: text/markdown
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.8**
+**Version 0.4.10**
 
 A simple and modular tool to evaluate any LLM application.
 
@@ -99,7 +99,7 @@ python -m moonshot -i moonshot-data -i moonshot-ui
 ```
 Check out our [Installation Guide](https://aiverify-foundation.github.io/moonshot/getting_started/quick_install/) for a more details.
 
-If you are having installation issues, see the [Troubleshooting Guide](https://aiverify-foundation.github.io/moonshot/common_issues/).
+If you are having installation issues, see the [Troubleshooting Guide](https://aiverify-foundation.github.io/moonshot/faq/).
 <details>
 <summary><b>Other installation options</b></summary>
 Here's a summary of other installation commands available:

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/README.md

@@ -2,7 +2,7 @@
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.8**
+**Version 0.4.10**
 
 A simple and modular tool to evaluate any LLM application.
 
@@ -55,7 +55,7 @@ python -m moonshot -i moonshot-data -i moonshot-ui
 ```
 Check out our [Installation Guide](https://aiverify-foundation.github.io/moonshot/getting_started/quick_install/) for a more details.
 
-If you are having installation issues, see the [Troubleshooting Guide](https://aiverify-foundation.github.io/moonshot/common_issues/).
+If you are having installation issues, see the [Troubleshooting Guide](https://aiverify-foundation.github.io/moonshot/faq/).
 <details>
 <summary><b>Other installation options</b></summary>
 Here's a summary of other installation commands available:

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/moonshot/__main__.py

@@ -116,36 +116,56 @@ def download_nltk_resources() -> None:
             raise
 
 
-def moonshot_data_installation() -> None:
-    # Code for moonshot-data installation
+def moonshot_data_installation(unattended: bool, overwrite: bool) -> None:
+    """
+    Install Moonshot Data from GitHub.
+
+    This function clones the Moonshot Data repository from GitHub, installs its requirements,
+    and sets up the necessary environment. If the target directory already exists, it handles
+    the situation based on the 'unattended' and 'overwrite' flags.
+
+    Args:
+        unattended (bool): If True, the function will not prompt the user for input.
+        overwrite (bool): If True, the existing directory will be removed before installation.
+    """
     logger.info("Installing Moonshot Data from GitHub")
     repo = "https://github.com/aiverify-foundation/moonshot-data.git"
     folder_name = repo.split("/")[-1].replace(".git", "")
+    do_install = True
 
-    # Check if the directory already exists
     if os.path.exists(folder_name):
-        logger.warning(f"Directory {folder_name} already exists.")
-        user_input = (
-            input(
-                f"Directory {folder_name} already exists. Do you want to remove it and reinstall? (Y/n): "
-            )
-            .strip()
-            .lower()
-        )
-        if user_input == "y":
-            logger.info(f"Removing directory {folder_name}.")
-            # Remove the existing directory
+        if overwrite:
+            logger.info(f"Removing directory {folder_name} due to --overwrite flag.")
             shutil.rmtree(folder_name)
         else:
-            logger.info("Exiting function without removing the directory.")
-            return
-
-    logger.info(f"Cloning {repo}")
-    # Clone the repository
-    run_subprocess(["git", "clone", repo], check=True)
-
-    # Create .env to point to installed folder
-    ms_lib_env_file(folder_name)
+            logger.warning(f"Directory {folder_name} already exists.")
+            if not unattended:
+                user_input = (
+                    input(
+                        f"Directory {folder_name} already exists. Do you want to remove it and reinstall? (Y/n): "
+                    )
+                    .strip()
+                    .lower()
+                )
+                if user_input == "y":
+                    logger.info(f"Removing directory {folder_name}.")
+                    shutil.rmtree(folder_name)
+                else:
+                    logger.info("Skipping removal of directory.")
+                    do_install = False
+            else:
+                do_install = False
+                logger.info(
+                    "Unattended mode detected. To reinstall, please use the --overwrite flag."
+                )
+
+    if do_install:
+        logger.info(f"Cloning {repo}")
+        # Clone the repository
+        run_subprocess(["git", "clone", repo], check=True)
+
+        # Create .env to point to installed folder
+        ms_lib_env_file(folder_name)
 
     # Change directory to the folder
     os.chdir(folder_name)
@@ -159,7 +179,6 @@ def moonshot_data_installation() -> None:
     # Change back to the base directory
     os.chdir("..")
 
-
 def check_node() -> bool:
     """
     Check if Node.js is installed on the user's machine.
@@ -176,36 +195,52 @@ def check_node() -> bool:
         return False
 
 
-def moonshot_ui_installation() -> None:
+def moonshot_ui_installation(unattended: bool, overwrite: bool) -> None:
+    """
+    Install Moonshot UI from GitHub.
+    """
     if not check_node():
         logger.error("Node.js is not installed. Please install Node.js to proceed.")
         return
 
-    # Code for moonshot-ui installation
+    logger.info("Installing Moonshot UI from GitHub")
     repo = "https://github.com/aiverify-foundation/moonshot-ui.git"
     folder_name = repo.split("/")[-1].replace(".git", "")
+    do_install = True
 
-    # Check if the directory already exists
     if os.path.exists(folder_name):
-        logger.warning(f"Directory {folder_name} already exists.")
-        user_input = (
-            input(
-                f"Directory {folder_name} already exists. Do you want to remove it and reinstall? (Y/n): "
-            )
-            .strip()
-            .lower()
-        )
-        if user_input == "y":
-            logger.info(f"Removing directory {folder_name}.")
-            # Remove the existing directory
+        if overwrite:
+            logger.info(f"Removing directory {folder_name} due to --overwrite flag.")
             shutil.rmtree(folder_name)
         else:
-            logger.info("Exiting function without removing the directory.")
-            return
-
-    logger.info(f"Cloning {repo}")
-    # Clone the repository
-    run_subprocess(["git", "clone", repo], check=True)
+            logger.warning(f"Directory {folder_name} already exists.")
+            if not unattended:
+                user_input = (
+                    input(
+                        f"Directory {folder_name} already exists. Do you want to remove it and reinstall? (Y/n): "
+                    )
+                    .strip()
+                    .lower()
+                )
+                if user_input == "y":
+                    logger.info(f"Removing directory {folder_name}.")
+                    shutil.rmtree(folder_name)
+                else:
+                    logger.info("Skipping removal of directory.")
+                    do_install = False
+            else:
+                do_install = False
+                logger.info(
+                    "Unattended mode detected. To reinstall, please use the --overwrite flag."
+                )
+
+    if do_install:
+        logger.info(f"Cloning {repo}")
+        # Clone the repository
+        run_subprocess(["git", "clone", repo], check=True)
+
+        # Create .env for UI
+        ms_ui_env_file(folder_name)
 
     # Change directory to the folder
     os.chdir(folder_name)
@@ -219,9 +254,6 @@ def moonshot_ui_installation() -> None:
     # Change back to the base directory
     os.chdir("..")
 
-    # Create .env for ui
-    ms_ui_env_file(folder_name)
-
 
 def run_moonshot_ui() -> None:
     """
@@ -262,15 +294,27 @@ def main() -> None:
     parser.add_argument(
         "-e", "--env", type=str, help="Path to the .env file", default=".env"
     )
+    parser.add_argument(
+        "-u",
+        "--unattended",
+        action="store_true",
+        help="Perform action without user interaction",
+    )
+    parser.add_argument(
+        "-o",
+        "--overwrite",
+        action="store_true",
+        help="Force delete and reinstall the specified module",
+    )
 
     args = parser.parse_args()
 
     # Handle installations based on the -i include arguments
     if "moonshot-data" in args.install:
-        moonshot_data_installation()
+        moonshot_data_installation(args.unattended, args.overwrite)
 
     if "moonshot-ui" in args.install:
-        moonshot_ui_installation()
+        moonshot_ui_installation(args.unattended, args.overwrite)
 
     # If mode is not specified, skip running any modes
     if args.mode is None:

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/moonshot/api.py

@@ -1,3 +1,11 @@
+from moonshot.src.api.api_bookmark import (
+    api_delete_all_bookmark,
+    api_delete_bookmark,
+    api_export_bookmarks,
+    api_get_all_bookmarks,
+    api_get_bookmark,
+    api_insert_bookmark,
+)
 from moonshot.src.api.api_connector import (
     api_create_connector_from_endpoint,
     api_create_connectors_from_endpoints,
@@ -26,10 +34,11 @@ from moonshot.src.api.api_cookbook import (
     api_update_cookbook,
 )
 from moonshot.src.api.api_dataset import (
+    api_convert_dataset,
     api_delete_dataset,
+    api_download_dataset,
     api_get_all_datasets,
     api_get_all_datasets_name,
-    api_create_datasets
 )
 from moonshot.src.api.api_environment_variables import api_set_environment_variables
 from moonshot.src.api.api_metrics import (
@@ -87,14 +96,6 @@ from moonshot.src.api.api_session import (
     api_update_prompt_template,
     api_update_system_prompt,
 )
-from moonshot.src.api.api_bookmark import (
-    api_get_all_bookmarks,
-    api_get_bookmark,
-    api_insert_bookmark,
-    api_delete_bookmark,
-    api_delete_all_bookmark,
-    api_export_bookmarks,
-)
 
 __all__ = [
     "api_create_connector_from_endpoint",
@@ -116,7 +117,8 @@ __all__ = [
     "api_read_cookbook",
     "api_read_cookbooks",
     "api_update_cookbook",
-    "api_create_datasets",
+    "api_convert_dataset",
+    "api_download_dataset",
     "api_delete_dataset",
     "api_get_all_datasets",
     "api_get_all_datasets_name",

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/moonshot/integrations/cli/benchmark/metrics.py

@@ -170,9 +170,15 @@ def _display_metrics(metrics_list: list):
     table.add_column("No.", width=2)
     table.add_column("Metric", justify="left", width=78)
     for idx, metric in enumerate(metrics_list, 1):
-        id, name, description, *other_args = metric.values()
+        id, name, description, endpoints, configurations, *other_args = metric.values()
         idx = metric.get("idx", idx)
-        result_info = f"[red]id: {id}[/red]\n\n[blue]{name}[/blue]\n{description}"
+        result_info = (
+            f"[red]Id: {id}[/red]\n\n"
+            f"[blue]Name: {name}[/blue]\n\n"
+            f"[blue]Description: {description}[/blue]\n\n"
+            f"[blue]Endpoints: {endpoints}[/blue]\n\n"
+            f"[blue]Configurations:{configurations}[/blue]\n\n"
+        )
 
         table.add_section()
         table.add_row(str(idx), result_info)

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/moonshot/integrations/cli/cli_errors.py

@@ -509,3 +509,17 @@ ERROR_RED_TEAMING_ADD_BOOKMARK_NO_ACTIVE_SESSION = (
 ERROR_RED_TEAMING_USE_BOOKMARK_NO_ACTIVE_SESSION = (
     "There is no active session. Activate a session to bookmark a prompt."
 )
+
+
+# ------------------------------------------------------------------------------
+# Redteaming - list_attack_modules
+# ------------------------------------------------------------------------------
+ERROR_RED_TEAMING_LIST_ATTACK_MODULES_FIND_VALIDATION = (
+    "Invalid type for parameter: find. Expecting type str."
+)
+ERROR_RED_TEAMING_LIST_ATTACK_MODULES_PAGINATION_VALIDATION = (
+    "Invalid type for parameter: pagination. Expecting type str."
+)
+ERROR_RED_TEAMING_LIST_ATTACK_MODULES_PAGINATION_VALIDATION_1 = (
+    "The 'pagination' argument must be a tuple of two integers."
+)

{aiverify_moonshot-0.4.8 → aiverify_moonshot-0.4.10}/moonshot/integrations/cli/common/common.py

@@ -2,10 +2,6 @@ import argparse
 
 import cmd2
 
-from moonshot.integrations.cli.common.dataset import (
-    add_dataset,
-    add_dataset_args
-)
 from moonshot.integrations.cli.common.connectors import (
     add_endpoint,
     add_endpoint_args,
@@ -20,6 +16,12 @@ from moonshot.integrations.cli.common.connectors import (
     view_endpoint,
     view_endpoint_args,
 )
+from moonshot.integrations.cli.common.dataset import (
+    convert_dataset,
+    convert_dataset_args,
+    download_dataset,
+    download_dataset_args,
+)
 from moonshot.integrations.cli.common.prompt_template import (
     delete_prompt_template,
     delete_prompt_template_args,
@@ -60,10 +62,14 @@ class CommonCommandSet(cmd2.CommandSet):
     def do_add_endpoint(self, args: argparse.Namespace) -> None:
         add_endpoint(args)
 
-    @cmd2.with_argparser(add_dataset_args)
-    def do_add_dataset(self, args:argparse.Namespace) -> None:
-        add_dataset(args)
-        
+    @cmd2.with_argparser(convert_dataset_args)
+    def do_convert_dataset(self, args: argparse.Namespace) -> None:
+        convert_dataset(args)
+
+    @cmd2.with_argparser(download_dataset_args)
+    def do_download_dataset(self, args: argparse.Namespace) -> None:
+        download_dataset(args)
+
     # ------------------------------------------------------------------------------
     # Delete contents
     # ------------------------------------------------------------------------------