aiverify-moonshot 0.4.0__tar.gz → 0.4.2__tar.gz

aiverify_moonshot-0.4.2/.github/pull_request_template.md

@@ -0,0 +1,88 @@
+## Description
+
+[Provide a brief description of the changes or features introduced by this pull request.]
+
+## Motivation and Context
+
+[Explain the motivation or the context behind this pull request. Why is it necessary?]
+
+## Type of Change
+
+<!-- Select the appropriate type of change: -->
+<!-- - feat: A new feature -->
+<!-- - fix: A bug fix -->
+<!-- - chore: Routine tasks, maintenance, or tooling changes -->
+<!-- - docs: Documentation updates -->
+<!-- - style: Code style changes (e.g., formatting, indentation) -->
+<!-- - refactor: Code refactoring without changes in functionality -->
+<!-- - test: Adding or modifying tests -->
+<!-- - perf: Performance improvements -->
+<!-- - ci: Changes to the CI/CD configuration or scripts -->
+<!-- - other: Other changes that don't fit into the above categories -->
+
+## How to Test
+
+[Provide clear instructions on how to test and verify the changes introduced by this pull request, including any specific unit tests you have created to demonstrate your changes.]
+
+## Checklist
+
+Please check all the boxes that apply to this pull request using "x":
+
+- [ ]  I have tested the changes locally and verified that they work as expected.
+- [ ]  I have added or updated the necessary documentation (README, API docs, etc.).
+- [ ]  I have added appropriate unit tests or functional tests for the changes made.
+- [ ]  I have followed the project's coding conventions and style guidelines.
+- [ ]  I have rebased my branch onto the latest commit of the main branch.
+- [ ]  I have squashed or reorganized my commits into logical units.
+- [ ]  I have added any necessary dependencies or packages to the project's build configuration.
+- [ ]  I have performed a self-review of my own code.
+- [ ]  I have read, understood and agree to the Developer Certificate of Origin below, which this project utilises.
+
+## Screenshots (if applicable)
+
+[If the changes involve visual modifications, include screenshots or GIFs that demonstrate the changes.]
+
+## Additional Notes
+
+[Add any additional information or context that might be relevant to reviewers.]
+
+<details>
+  <summary>Developer Certificate of Origin</summary>
+
+ ```
+Developer Certificate of Origin
+Version 1.1
+
+Copyright (C) 2004, 2006 The Linux Foundation and its contributors.
+
+Everyone is permitted to copy and distribute verbatim copies of this
+license document, but changing it is not allowed.
+
+
+Developer's Certificate of Origin 1.1
+
+By making a contribution to this project, I certify that:
+
+(a) The contribution was created in whole or in part by me and I
+    have the right to submit it under the open source license
+    indicated in the file; or
+
+(b) The contribution is based upon previous work that, to the best
+    of my knowledge, is covered under an appropriate open source
+    license and I have the right under that license to submit that
+    work with modifications, whether created in whole or in part
+    by me, under the same open source license (unless I am
+    permitted to submit under a different license), as indicated
+    in the file; or
+
+(c) The contribution was provided directly to me by some other
+    person who certified (a), (b) or (c) and I have not modified
+    it.
+
+(d) I understand and agree that this project and the contribution
+    are public and that a record of the contribution (including all
+    personal information I submit with it, including my sign-off) is
+    maintained indefinitely and may be redistributed consistent with
+    this project or the open source license(s) involved.
+ ```
+</details>

aiverify_moonshot-0.4.2/.github/workflows/sca_scan.yaml

@@ -0,0 +1,64 @@
+name: Software Composition Analysis
+
+# Runs when a pull request review is being submitted
+on: 
+  pull_request:
+
+  # Run this workflow manually from Actions tab
+  workflow_dispatch:
+    inputs:
+      branch_to_test:
+        description: 'Branch or tag to run test'
+        required: true
+        default: 'main'
+        type: string
+
+# Allow one concurrent deployment
+concurrency:
+  group: ${{ github.repository }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  software-composition-analysis:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+
+    - name: Checkout Code (Pull_Request)
+      # if: github.event_name == 'pull_request'
+      uses: actions/checkout@v3
+      with:
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          submodules: recursive
+
+    - name: Checkout Code (Workflow_Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      uses: actions/checkout@v3
+      with:
+          ref: ${{ inputs.branch_to_test }}
+
+    - name: Set Branch Variable (Pull_Request)
+      # if: github.event_name == 'pull_request'
+      run: |
+          echo "BRANCH=branch" >> "$GITHUB_ENV"
+
+    - name: Set Branch Variable (Workflow_Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      run: |
+          echo "BRANCH=${{ inputs.branch_to_test }}" >> "$GITHUB_ENV"
+
+    - name: Setup Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: "3.11"
+
+    - name: Install & Scan MOONSHOT
+      run: |
+          python -m venv venv
+          source venv/bin/activate
+          pip install -r requirements.txt
+          bash <(curl -s -L https://detect.synopsys.com/detect9.sh) --blackduck.url=https://blackduck.aipo-imda.net/ --blackduck.api.token=${{ secrets.BLACKDUCK_TOKEN }} --detect.detector.search.depth=10 --detect.project.name=aiverify-moonshot --detect.project.version.name=$BRANCH --detect.source.path=. --detect.code.location.name="aiverify-moonshot_$BRANCH" --detect.excluded.directories=/tests --blackduck.trust.cert=true --detect.excluded.detector.types=POETRY
+          cd ../

aiverify_moonshot-0.4.2/.github/workflows/smoke-test.yaml

@@ -0,0 +1,127 @@
+# Smoke Test
+
+name: Moonshot Smoke Test
+
+on:
+  # Runs on Pull Request
+  pull_request:
+  #  types: [submitted]
+  #  branches:
+  #    - 'main'
+
+  # Run this workflow manually from Actions tab
+  workflow_dispatch:
+    inputs:
+      moonshot_branch:
+        description: 'Moonshot Branch / Tag Name'
+        required: true
+        default: 'main'
+        type: string
+      moonshot_data_branch:
+        description: 'Moonshot Data Branch / Tag Name'
+        required: true
+        default: 'main'
+        type: string
+      moonshot_ui_branch:
+        description: 'Moonshot UI Branch / Tag Name'
+        required: true
+        default: 'main'
+        type: string
+
+# Allow one concurrent deployment
+concurrency:
+  group: ${{ github.repository }}-${{ github.workflow }}
+  cancel-in-progress: true
+
+jobs:
+  smoke-test:
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+
+    steps:
+
+    - name: Checkout Moonshot (Pull Request)
+      if: github.event_name == 'pull_request'
+      uses: actions/checkout@v4
+      with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.ref }}
+
+    - name: Checkout Moonshot (Workflow Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      uses: actions/checkout@v4
+      with:
+          repository: aiverify-foundation/moonshot-data
+          ref: ${{ inputs.moonshot_branch }}
+    
+    - name: Setup Python 3.11
+      uses: actions/setup-python@v4
+      with:
+          python-version: '3.11'
+    
+    - name: Setup Moonshot
+      run: |
+        python -m venv venv
+        source venv/bin/activate
+        pip install -r requirements.txt
+
+    - name: Checkout Moonshot Data (Pull Request)
+      if: github.event_name == 'pull_request'
+      uses: actions/checkout@v4
+      with: 
+         repository: aiverify-foundation/moonshot-data
+         ref: ${{ vars.MOONSHOT_DATA_BRANCH }}
+         path: moonshot-data
+
+    - name: Checkout Moonshot Data (Workflow Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      uses: actions/checkout@v4
+      with: 
+         repository: aiverify-foundation/moonshot-data
+         ref: ${{ inputs.moonshot_data_branch }}
+         path: moonshot-data
+
+    - name: Setup Moonshot Data
+      run: |
+        source venv/bin/activate
+        cd moonshot-data
+        pip install -r requirements.txt
+
+    - name: Checkout Moonshot UI (Pull Request)
+      if: github.event_name == 'pull_request'
+      uses: actions/checkout@v4
+      with:
+        repository: aiverify-foundation/moonshot-ui
+        ref: ${{ vars.MOONSHOT_UI_BRANCH }}
+        path: moonshot-ui
+
+    - name: Checkout Moonshot UI (Workflow Dispatch)
+      if: github.event_name == 'workflow_dispatch'
+      uses: actions/checkout@v4
+      with:
+        repository: aiverify-foundation/moonshot-ui
+        ref: ${{ inputs.moonshot_ui_branch }}
+        path: moonshot-ui
+    
+    - name: Setup Moonshot UI
+      run: |
+        cd moonshot-ui
+        npm ci
+        npm run build
+        cd ../
+        source venv/bin/activate
+        python -m moonshot web &
+
+    - name: Checkout Smoke Test 
+      uses: actions/checkout@v4
+      with:
+          repository: aiverify-foundation/moonshot-smoke-testing
+          path: moonshot-smoke-testing
+
+    - name: Run Smoke Test
+      run: |
+         cd moonshot-smoke-testing
+         npm ci
+         npx playwright install --with-deps
+         npx playwright test tests/main-page.spec.ts

{aiverify_moonshot-0.4.0 → aiverify_moonshot-0.4.2}/.pre-commit-config.yaml

@@ -17,4 +17,8 @@ repos:
     hooks:
       - id: flake8
         name: flake8 - Python linting
-        exclude: ^tests
+        exclude: ^tests
+-   repo: https://github.com/gitleaks/gitleaks
+    rev: v8.18.4
+    hooks:
+      - id: gitleaks

{aiverify_moonshot-0.4.0 → aiverify_moonshot-0.4.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: aiverify-moonshot
-Version: 0.4.0
+Version: 0.4.2
 Summary: AI Verify advances Gen AI testing with Project Moonshot.
 Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
 Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
@@ -43,7 +43,7 @@ Description-Content-Type: text/markdown
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.0**
+**Version 0.4.2**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.0 → aiverify_moonshot-0.4.2}/README.md

@@ -2,7 +2,7 @@
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.0**
+**Version 0.4.2**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.0 → aiverify_moonshot-0.4.2}/moonshot/__main__.py

@@ -1,15 +1,24 @@
-import sys
-import warnings
 import argparse
+import os
 import platform
 import subprocess
-import os
-import threading 
+import sys
+import threading
+import warnings
+
 from dotenv import dotenv_values
+
 from moonshot.api import api_set_environment_variables
+from moonshot.src.utils.log import configure_logger
+
+# Create a logger for this module
+logger = configure_logger(__name__)
+
 """
 Run the Moonshot application
 """
+
+
 def run_subprocess(*args, **kwargs):
     """
     Run a subprocess with the option to use shell=True on Windows.
@@ -18,6 +27,7 @@ def run_subprocess(*args, **kwargs):
         kwargs["shell"] = True
     return subprocess.run(*args, **kwargs)
 
+
 def ms_lib_env_file(data_repo_name):
     """
     Writes the env file to be used for moonshot library
@@ -25,6 +35,7 @@ def ms_lib_env_file(data_repo_name):
     env_content_data = f"""
     # For Data
     ATTACK_MODULES="./{data_repo_name}/attack-modules"
+    BOOKMARKS="./{data_repo_name}/generated-outputs/bookmarks"
     CONNECTORS="./{data_repo_name}/connectors"
     CONNECTORS_ENDPOINTS="./{data_repo_name}/connectors-endpoints"
     CONTEXT_STRATEGY="./{data_repo_name}/context-strategy"
@@ -40,7 +51,7 @@ def ms_lib_env_file(data_repo_name):
     RESULTS_MODULES="./{data_repo_name}/results-modules"
     RUNNERS="./{data_repo_name}/generated-outputs/runners"
     RUNNERS_MODULES="./{data_repo_name}/runners-modules"
-    TOKENIZERS_PARALLELISM = false 
+    TOKENIZERS_PARALLELISM = false
     """
 
     env_content_web_api = """
@@ -58,6 +69,7 @@ def ms_lib_env_file(data_repo_name):
         combined_content = env_content_data + env_content_web_api
         env_file.write(combined_content.strip())
 
+
 def ms_ui_env_file(ui_repo):
     """
     Write the env file to be used with moonshot ui
@@ -71,66 +83,73 @@ def ms_ui_env_file(ui_repo):
     with open(os.path.join(ui_repo, ".env"), "w") as env_file:
         env_file.write(env_content.strip())
 
+
 def moonshot_data_installation():
     # Code for moonshot-data installation
-    print("Installing Moonshot Data from GitHub")
+    logger.info("Installing Moonshot Data from GitHub")
     repo = "https://github.com/aiverify-foundation/moonshot-data.git"
     folder_name = repo.split("/")[-1].replace(".git", "")
-    
+
     # Check if the directory already exists
     if not os.path.exists(folder_name):
-        print(f"Cloning {repo}")
+        logger.info(f"Cloning {repo}")
         # Clone the repository
         run_subprocess(["git", "clone", repo], check=True)
-        
+
+        # Create .env to point to installed folder
+        ms_lib_env_file(folder_name)
+
         # Change directory to the folder
         os.chdir(folder_name)
 
-        print(f"Installing requirements for {folder_name}")
+        logger.info(f"Installing requirements for {folder_name}")
         # Install the requirements if they exist
         if os.path.exists("requirements.txt"):
             run_subprocess(["pip", "install", "-r", "requirements.txt"], check=True)
             import nltk
-            nltk.download('punkt')
-            nltk.download('stopwords')
-            nltk.download('averaged_perceptron_tagger')
-            nltk.download('universal_tagset')
-        
+
+            nltk.download("punkt")
+            nltk.download("stopwords")
+            nltk.download("averaged_perceptron_tagger")
+            nltk.download("universal_tagset")
+
         # Change back to the base directory
         os.chdir("..")
 
-        # Create .env to point to installed folder
-        ms_lib_env_file(folder_name)
     else:
-        print(f"Directory {folder_name} already exists, skipping clone.")
+        logger.warning(f"Directory {folder_name} already exists, skipping clone.")
+
 
 def moonshot_ui_installation():
     # Code for moonshot-ui installation
     repo = "https://github.com/aiverify-foundation/moonshot-ui.git"
     folder_name = repo.split("/")[-1].replace(".git", "")
-    
+
     # Check if the directory already exists
     if not os.path.exists(folder_name):
-        print(f"Cloning {repo}")
+        logger.info(f"Cloning {repo}")
         # Clone the repository
         run_subprocess(["git", "clone", repo], check=True)
 
         # Change directory to the folder
         os.chdir(folder_name)
 
-        print(f"Installing requirements for {folder_name}")        
+        logger.info(f"Installing requirements for {folder_name}")
         # Install the requirements if they exist
         if os.path.exists("package.json"):
             run_subprocess(["npm", "install"], check=True)
             run_subprocess(["npm", "run", "build"], check=True)
-        
+
         # Change back to the base directory
         os.chdir("..")
 
         # Create .env for ui
         ms_ui_env_file(folder_name)
     else:
-        print(f"Directory {folder_name} already exists, skipping installation.")
+        logger.warning(
+            f"Directory {folder_name} already exists, skipping installation."
+        )
+
 
 def run_moonshot_ui_dev():
     """
@@ -140,31 +159,51 @@ def run_moonshot_ui_dev():
     ui_dev_dir = os.path.join(base_directory, "moonshot-ui")
 
     if not os.path.exists(ui_dev_dir):
-        print("moonshot-ui does not exist. Please run with '-i moonshot-ui' to install moonshot-ui first.")
+        logger.error(
+            "moonshot-ui does not exist. Please run with '-i moonshot-ui' to install moonshot-ui first."
+        )
         sys.exit(1)
     # ms_ui_env_file(ui_dev_dir)
-    run_subprocess(['npm', 'start'], cwd=ui_dev_dir)
+    run_subprocess(["npm", "start"], cwd=ui_dev_dir)
+
 
 def main():
     parser = argparse.ArgumentParser(description="Run the Moonshot application")
-    parser.add_argument('mode', nargs='?', choices=['web-api', 'cli', 'web', 'help'], help='Mode to run Moonshot in', default=help)
-    parser.add_argument('cli_command', nargs='?', help='The CLI command to run (e.g., "interactive")')
-    parser.add_argument('-i', '--install', action='append', choices=['moonshot-data', 'moonshot-ui'], help='Modules to install', default=[])
-    parser.add_argument('-e', '--env', type=str, help='Path to the .env file', default='.env')
-    
+    parser.add_argument(
+        "mode",
+        nargs="?",
+        choices=["web-api", "cli", "web", "help"],
+        help="Mode to run Moonshot in",
+        default=help,
+    )
+    parser.add_argument(
+        "cli_command", nargs="?", help='The CLI command to run (e.g., "interactive")'
+    )
+    parser.add_argument(
+        "-i",
+        "--install",
+        action="append",
+        choices=["moonshot-data", "moonshot-ui"],
+        help="Modules to install",
+        default=[],
+    )
+    parser.add_argument(
+        "-e", "--env", type=str, help="Path to the .env file", default=".env"
+    )
+
     args = parser.parse_args()
-    
+
     # Handle installations based on the -i include arguments
-    if 'moonshot-data' in args.install:
+    if "moonshot-data" in args.install:
         moonshot_data_installation()
 
-    if 'moonshot-ui' in args.install:
+    if "moonshot-ui" in args.install:
         moonshot_ui_installation()
 
     # If mode is not specified, skip running any modes
     if args.mode is None:
         return
-    
+
     if args.mode == "help":
         parser.print_help()
         sys.exit(1)
@@ -173,18 +212,21 @@ def main():
 
     if args.mode == "web-api":
         from moonshot.integrations.web_api import __main__ as web_api
+
         web_api.start_app()
     elif args.mode == "web":
-    # Create and start the UI dev server thread
+        # Create and start the UI dev server thread
         ui_thread = threading.Thread(target=run_moonshot_ui_dev)
         ui_thread.start()
         ui_thread.join(timeout=0.1)  # Wait briefly for the thread to become alive
         if not ui_thread.is_alive():
             sys.exit(1)
         from moonshot.integrations.web_api import __main__ as web_api
+
         web_api.start_app()
     elif args.mode == "cli":
         from moonshot.integrations.cli import __main__ as cli
+
         cli.start_app(args.cli_command)
 
         # Handle CLI mode here, possibly also with additional arguments
@@ -193,6 +235,7 @@ def main():
         parser.print_help()
         sys.exit(1)
 
+
 if __name__ == "__main__":
     warnings.filterwarnings("ignore")
-    main()
+    main()

{aiverify_moonshot-0.4.0 → aiverify_moonshot-0.4.2}/moonshot/api.py

@@ -86,6 +86,14 @@ from moonshot.src.api.api_session import (
     api_update_prompt_template,
     api_update_system_prompt,
 )
+from moonshot.src.api.api_bookmark import (
+    api_get_all_bookmarks,
+    api_get_bookmark,
+    api_insert_bookmark,
+    api_delete_bookmark,
+    api_delete_all_bookmark,
+    api_export_bookmarks,
+)
 
 __all__ = [
     "api_create_connector_from_endpoint",
@@ -152,4 +160,10 @@ __all__ = [
     "api_update_metric",
     "api_update_prompt_template",
     "api_update_system_prompt",
+    "api_get_all_bookmarks",
+    "api_get_bookmark",
+    "api_insert_bookmark",
+    "api_delete_bookmark",
+    "api_delete_all_bookmark",
+    "api_export_bookmarks",
 ]

{aiverify_moonshot-0.4.0 → aiverify_moonshot-0.4.2}/moonshot/integrations/cli/benchmark/benchmark.py

@@ -8,6 +8,7 @@ from moonshot.integrations.cli.benchmark.cookbook import (
     delete_cookbook,
     delete_cookbook_args,
     list_cookbooks,
+    list_cookbooks_args,
     run_cookbook,
     run_cookbook_args,
     update_cookbook,
@@ -19,6 +20,7 @@ from moonshot.integrations.cli.benchmark.datasets import (
     delete_dataset,
     delete_dataset_args,
     list_datasets,
+    list_datasets_args,
     view_dataset,
     view_dataset_args,
 )
@@ -26,6 +28,7 @@ from moonshot.integrations.cli.benchmark.metrics import (
     delete_metric,
     delete_metric_args,
     list_metrics,
+    list_metrics_args,
     view_metric,
     view_metric_args,
 )
@@ -35,6 +38,7 @@ from moonshot.integrations.cli.benchmark.recipe import (
     delete_recipe,
     delete_recipe_args,
     list_recipes,
+    list_recipes_args,
     run_recipe,
     run_recipe_args,
     update_recipe,
@@ -46,10 +50,16 @@ from moonshot.integrations.cli.benchmark.result import (
     delete_result,
     delete_result_args,
     list_results,
+    list_results_args,
     view_result,
     view_result_args,
 )
-from moonshot.integrations.cli.benchmark.run import list_runs, view_run, view_run_args
+from moonshot.integrations.cli.benchmark.run import (
+    list_runs,
+    list_runs_args,
+    view_run,
+    view_run_args,
+)
 from moonshot.integrations.cli.benchmark.runner import (
     delete_runner,
     delete_runner_args,
@@ -68,26 +78,32 @@ class BenchmarkCommandSet(cmd2.CommandSet):
     # List contents
     # ------------------------------------------------------------------------------
 
-    def do_list_cookbooks(self, _: cmd2.Statement) -> None:
-        list_cookbooks()
+    @cmd2.with_argparser(list_cookbooks_args)
+    def do_list_cookbooks(self, args: argparse.Namespace) -> None:
+        list_cookbooks(args)
 
-    def do_list_recipes(self, _: cmd2.Statement) -> None:
-        list_recipes()
+    @cmd2.with_argparser(list_recipes_args)
+    def do_list_recipes(self, args: argparse.Namespace) -> None:
+        list_recipes(args)
 
-    def do_list_results(self, _: cmd2.Statement) -> None:
-        list_results()
+    @cmd2.with_argparser(list_results_args)
+    def do_list_results(self, args: argparse.Namespace) -> None:
+        list_results(args)
 
     def do_list_runners(self, _: cmd2.Statement) -> None:
         list_runners()
 
-    def do_list_runs(self, _: cmd2.Statement) -> None:
-        list_runs()
+    @cmd2.with_argparser(list_runs_args)
+    def do_list_runs(self, args: argparse.Namespace) -> None:
+        list_runs(args)
 
-    def do_list_metrics(self, _: cmd2.Statement) -> None:
-        list_metrics()
+    @cmd2.with_argparser(list_metrics_args)
+    def do_list_metrics(self, args: argparse.Namespace) -> None:
+        list_metrics(args)
 
-    def do_list_datasets(self, _: cmd2.Statement) -> None:
-        list_datasets()
+    @cmd2.with_argparser(list_datasets_args)
+    def do_list_datasets(self, args: argparse.Namespace) -> None:
+        list_datasets(args)
 
     # ------------------------------------------------------------------------------
     # Add contents