aiverify-moonshot 0.4.11__tar.gz → 0.5.0__tar.gz

aiverify_moonshot-0.5.0/.github/developing-workflows.md

@@ -0,0 +1,28 @@
+Developing GitHub Workflows
+===========================
+Follow these instructions to develop Github workflows in a non-main branch 
+instead of in the main branch to avoid creating unwanted commit records to
+the main branch.
+
+Workflows created in a non-main branch can only be run via CLI as follows:
+
+`gh workflow run <workflow-name> --ref <branch-name> -f <parameter-name>=<parameter-value>`
+
+For example, to run a workflow called License File Generation created in the ci-workflow-dev branch:
+
+`gh workflow run 'License File Generation' --ref ci-workflow-dev -f repo=aiverify-foundation/moonshot -f branch=ci-workflow-dev`
+
+To run the workflow via CLI, it must appear in the workflow list when you
+run the following command:
+
+`gh workflow list`
+
+If it is not in the list, you have to trigger it once, this can be done by
+creating a trigger for push in your workflow as follows and pushing a commit
+to the branch. Once the workflow appears in the workflow list, you can remove
+the trigger for push from your workflow.
+
+    on:  
+        push:  
+            branches: [ "ci-workflow-dev" ]  
+

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/.github/workflows/docs-update.yaml

@@ -37,7 +37,7 @@ jobs:
       - name: Checkout code
         uses: actions/checkout@v3
         with:
-          ref: 'docs'
+          ref: ${{ inputs.docs_branch }}
 
       - name: Setup python 3.11
         uses: actions/setup-python@v5
@@ -53,7 +53,7 @@ jobs:
         run: |
           mkdocs build
 
-      - name: Upload artifact
+      - name: Upload docs artifact
         uses: actions/upload-pages-artifact@v3
         with:
           path: ./site
@@ -66,7 +66,8 @@ jobs:
       pages: write      # to deploy to Pages
       id-token: write   # to verify the deployment originates from an appropriate source
 
-    # Deploy to the github-pages environment (ensure main can deploy to pages)
+    # Deploy to the github-pages environment
+    # Ensure main branch can deploy to pages in the GitHub Pages Environment settings
     environment:
       name: github-pages
       url: ${{ steps.deployment.outputs.page_url }}

aiverify_moonshot-0.5.0/.github/workflows/gh-event-notification.yaml

@@ -0,0 +1,50 @@
+name: GH Event Notification
+
+on:
+  issues:
+    types: [opened]
+  pull_request:
+    types: [opened]
+
+jobs:
+  notify:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Get Issue details
+        if: ${{ github.event_name == 'issues' }}
+        id: issue_details
+        env:
+          TITLE: ${{ github.event.issue.title }}
+        run: |
+          echo "type=issue" >> $GITHUB_ENV
+          echo "title=$TITLE" >> $GITHUB_ENV
+          echo "user=${{ github.event.issue.user.login }}" >> $GITHUB_ENV
+          echo "url=${{ github.event.issue.html_url }}" >> $GITHUB_ENV
+          echo "repo=${{ github.repository }}" >> $GITHUB_ENV
+
+      - name: Get PR details
+        if: ${{ github.event_name == 'pull_request' }}
+        id: pr_details
+        env:
+          TITLE: ${{ github.event.pull_request.title }}
+        run: |
+          echo "type=pull_request" >> $GITHUB_ENV
+          echo "title=$TITLE" >> $GITHUB_ENV
+          echo "user=${{ github.event.pull_request.user.login }}" >> $GITHUB_ENV
+          echo "url=${{ github.event.pull_request.html_url }}" >> $GITHUB_ENV
+          echo "repo=${{ github.repository }}" >> $GITHUB_ENV
+
+      - name: Send notification to Slack
+        uses: slackapi/slack-github-action@v1.26.0
+        with:
+          payload: |
+            {
+              "type": "${{ env.type }}",
+              "title": "${{ env.title }}",
+              "user": "${{ env.user }}",
+              "repo": "${{ env.repo }}",
+              "url": "${{ env.url }}"
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_GH_NOTIF_URL }}

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/.github/workflows/integration-test.yaml

@@ -36,7 +36,7 @@ jobs:
   integration-test:
 
     runs-on: ubuntu-latest
-    timeout-minutes: 20
+    timeout-minutes: 120
 
     steps:
 
@@ -104,6 +104,13 @@ jobs:
         repository: aiverify-foundation/moonshot-ui
         ref: ${{ inputs.moonshot_ui_branch }}
         path: moonshot-ui
+
+     # Download NLTK stopwords
+    - name: Download NLTK stopwords
+      run: |
+        source venv/bin/activate
+        pip install nltk
+        python -c "import nltk; nltk.download('stopwords')"
     
     - name: Setup Moonshot UI
       run: |
@@ -122,14 +129,22 @@ jobs:
 
     - name: Run Integration Test
       env: 
-        AZURE_OPENAI_URI: ${{ secrets.AZURE_OPENAI_URI }}
-        AZURE_OPENAI_TOKEN: ${{ secrets.AZURE_OPENAI_TOKEN }}
+        URI: ${{ secrets.URI }}
+        TOKEN: ${{ secrets.TOKEN }}
+        URI2: ${{ secrets.URI2 }}
+        TOKEN2: ${{ secrets.TOKEN2 }}
         ADDITIONAL_PARAMETERS: ${{ secrets.ADDITIONAL_PARAMETERS }}
+        TOGETHER_TOKEN: ${{ secrets.TOGETHER_TOKEN }}
+        OPENAI_TOKEN: ${{ secrets.OPENAI_TOKEN }}
+        AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+        AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+        GOOGLE_TOKEN: ${{ secrets.GOOGLE_TOKEN }}
       run: |
          cd moonshot-integration-testing/ui-integration-testing
          npm ci
+         npm install dotenv
          npx playwright install --with-deps
-         AZURE_OPENAI_URI="$AZURE_OPENAI_URI" AZURE_OPENAI_TOKEN="$AZURE_OPENAI_TOKEN" ADDITIONAL_PARAMETERS="$ADDITIONAL_PARAMETERS" npx playwright test
+         URI="$URI" TOKEN="$TOKEN" URI2="$URI2" TOKEN2="$TOKEN2" ADDITIONAL_PARAMETERS="$ADDITIONAL_PARAMETERS" TOGETHER_TOKEN="$TOGETHER_TOKEN" OPENAI_TOKEN="$OPENAI_TOKEN" AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" GOOGLE_TOKEN="$GOOGLE_TOKEN" npx playwright test
 
     - name: TestRail CLI upload results
       env:

aiverify_moonshot-0.5.0/.github/workflows/notices-file-gen.yaml

@@ -0,0 +1,82 @@
+name: Notices File Generation
+
+on:
+  workflow_call:
+    inputs:
+      repo:
+        description: 'Repository name'
+        required: true
+        default: 'aiverify-foundation/moonshot'
+        type: string
+      branch:
+        description: 'Branch or tag'
+        required: true
+        default: 'main'
+        type: string
+
+  workflow_dispatch:
+    inputs:
+      repo:
+        description: 'Repository name'
+        required: true
+        default: 'aiverify-foundation/moonshot'
+        type: string
+      branch:
+        description: 'Branch or tag'
+        required: true
+        default: 'main'
+        type: string
+
+jobs:
+  notices-file-gen:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.inputs.repo }}
+          ref: ${{ github.event.inputs.branch }}
+          submodules: recursive
+
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Generate notices file
+        run: |
+          set +e
+          pip install -r requirements.txt
+          npm install
+          repo_name=$(echo "${{ github.event.inputs.repo }}" | awk -F'/' '{print $2}')
+          bash <(curl -s -L https://detect.synopsys.com/detect9.sh) \
+            --blackduck.url=https://blackduck.aipo-imda.net/ \
+            --blackduck.api.token=${{ secrets.BLACKDUCK_TOKEN }} \
+            --detect.detector.search.depth=10 \
+            --detect.project.name=aiverify-${repo_name} \
+            --detect.project.version.name=${{ github.event.inputs.branch }} \
+            --detect.source.path=. \
+            --detect.code.location.name="aiverify-${repo_name}_${{ github.event.inputs.branch }}" \
+            --detect.excluded.directories=/tests \
+            --blackduck.trust.cert=true \
+            --detect.excluded.detector.types=POETRY \
+            --detect.blackduck.scan.mode=RAPID \
+            --detect.notices.report=true \
+            --detect.notices.report.path=.
+          set -e
+          ls -al
+          cat ./*_Black_Duck_Notices_Report.txt
+
+#
+#      - name: Commit notices file
+#        run: |
+#          set +e
+#          mv aiverify_moonshot_main_Black_Duck_Notices_Report.txt NOTICES.md
+#          git add NOTICES.md
+#          git config user.name "imda-btg"
+#          git config user.email "idma-btg@imda.gov.sg"
+#          git commit -m "update notices file"
+#          git push
+#          set -e
+#

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/.github/workflows/pre-build-checks.yaml

@@ -3,7 +3,7 @@
 # 2. Code quality analysis (flake8)
 # 3. Dependency analysis (vulnerabilities)
 # 4. Dependency analysis (undesirable licenses)
-# 5. Deploy reports generated from the above to GitHub Pages
+# 5. Send slack notification on check results
 
 name: Pre-Build Checks
 
@@ -50,8 +50,9 @@ jobs:
         if: github.event_name == 'workflow_dispatch'
         run: |
           echo "MS_BRANCH=${{ inputs.branch_to_test }}" >> "$GITHUB_ENV"
-          echo "MS_DATA_BRANCH=${{ inputs.branch_to_test }}" >> "$GITHUB_ENV"
+          echo "MS_DATA_BRANCH=dev_main" >> "$GITHUB_ENV"
           echo "PR_NUM=#0" >> "$GITHUB_ENV"
+          echo "${{ inputs.branch_to_test }}"
 
       # Checkout moonshot code
       - name: Checkout code (moonshot)
@@ -110,7 +111,7 @@ jobs:
           HOST_PORT=5000" > .env
           cat .env
 
-      # Unit tests & coverage
+      # Check 1: Unit tests & coverage
       - name: Unit tests with coverage
         id: unit_tests
         if: ${{ ! cancelled() }}
@@ -129,7 +130,7 @@ jobs:
             exit 1
           fi
 
-      # Code quality analysis (flake8)
+      # Check 2: Code quality analysis (flake8)
       - name: Code quality analysis (flake8)
         id: code_quality
         if: ${{ ! cancelled() }}
@@ -142,7 +143,7 @@ jobs:
           set -e
           exit $lint_status
 
-      # pip-audit
+      # Check 3: Dependency vulnerability & license analysis (pip-audit)
       - name: Dependency analysis (vulnerabilities & licenses)
         id: dependency_analysis
         if: ${{ ! cancelled() }}

aiverify_moonshot-0.4.11/.github/workflows/nightly-build.yaml → aiverify_moonshot-0.5.0/.github/workflows/sit-build.yaml

@@ -1,5 +1,5 @@
 
-name: Nightly Build
+name: SIT Build
 
 on:
 
@@ -30,7 +30,8 @@ concurrency:
 
 jobs:
 
-  nightly-build:
+  # Remote deployment of moonshot to SIT server
+  sit-build:
 
     runs-on: ubuntu-latest
     timeout-minutes: 60
@@ -125,56 +126,62 @@ jobs:
             --output text)
           echo "Command status: $status"
 
-      - name: Run smoke test
-        timeout-minutes: 30
+#      - name: Run smoke test
+#        timeout-minutes: 30
+#        run: |
+#          command_id=$(aws ssm send-command \
+#            --instance-ids "${{ vars.AWS_SIT_INSTANCE_ID }}" \
+#            --document-name "AWS-RunShellScript" \
+#            --comment "Run smoke test" \
+#            --parameters 'commands=["sudo su - ubuntu -c \". ~/scripts/run_smoke_test.sh ${{ secrets.AZURE_OPENAI_TOKEN }} ${{ secrets.AZURE_OPENAI_URI }} \""]' \
+#            --query "Command.CommandId" \
+#            --output text)
+#
+#          max_attempts=20
+#          delay=60
+#          attempts=0
+#
+#          while [ $attempts -lt $max_attempts ]; do
+#            status=$(aws ssm list-command-invocations \
+#              --command-id "$command_id" \
+#              --instance-id "${{ vars.AWS_SIT_INSTANCE_ID }}" \
+#              --query "CommandInvocations[0].Status" \
+#              --output text)
+#
+#            if [ "$status" == "Success" ] || [ "$status" == "Failed" ]; then
+#              output=$(aws ssm get-command-invocation \
+#                --command-id "$command_id" \
+#                --instance-id "${{ vars.AWS_SIT_INSTANCE_ID }}" \
+#                --query "StandardOutputContent" \
+#                --output text)
+#              echo "SMOKE_TEST_STATUS=$status" >> $GITHUB_ENV
+#              echo "Smoke test output: $output"
+#              echo "Command status: $status"
+#              if [ "$status" == "Success" ]; then
+#                exit 0
+#              else
+#                exit 1
+#              fi
+#            elif [ "$status" == "InProgress" ]; then
+#              echo "Command status: $status, waiting..."
+#              sleep $delay
+#              attempts=$((attempts + 1))
+#            else
+#              echo "Unexpected command status: $status"
+#              echo "SMOKE_TEST_STATUS=$status" >> $GITHUB_ENV
+#              exit 1
+#            fi
+#          done
+#
+#          echo "Max attempts reached, exiting."
+#          echo "SMOKE_TEST_STATUS=timeout" >> $GITHUB_ENV
+#          exit 1
+
+      # Remote execution of integration test in SIT server
+      - name: Run integration test
         run: |
-          command_id=$(aws ssm send-command \
-            --instance-ids "${{ vars.AWS_SIT_INSTANCE_ID }}" \
-            --document-name "AWS-RunShellScript" \
-            --comment "Run smoke test" \
-            --parameters 'commands=["sudo su - ubuntu -c \". ~/scripts/run_smoke_test.sh ${{ secrets.AZURE_OPENAI_TOKEN }} ${{ secrets.AZURE_OPENAI_URI }} \""]' \
-            --query "Command.CommandId" \
-            --output text)
-      
-          max_attempts=20
-          delay=60
-          attempts=0
-      
-          while [ $attempts -lt $max_attempts ]; do
-            status=$(aws ssm list-command-invocations \
-              --command-id "$command_id" \
-              --instance-id "${{ vars.AWS_SIT_INSTANCE_ID }}" \
-              --query "CommandInvocations[0].Status" \
-              --output text)
-      
-            if [ "$status" == "Success" ] || [ "$status" == "Failed" ]; then
-              output=$(aws ssm get-command-invocation \
-                --command-id "$command_id" \
-                --instance-id "${{ vars.AWS_SIT_INSTANCE_ID }}" \
-                --query "StandardOutputContent" \
-                --output text)
-              echo "SMOKE_TEST_STATUS=$status" >> $GITHUB_ENV
-              echo "Smoke test output: $output"
-              echo "Command status: $status"
-              if [ "$status" == "Success" ]; then
-                exit 0
-              else
-                exit 1
-              fi
-            elif [ "$status" == "InProgress" ]; then
-              echo "Command status: $status, waiting..."
-              sleep $delay
-              attempts=$((attempts + 1))
-            else
-              echo "Unexpected command status: $status"
-              echo "SMOKE_TEST_STATUS=$status" >> $GITHUB_ENV
-              exit 1
-            fi
-          done
-      
-          echo "Max attempts reached, exiting."
-          echo "SMOKE_TEST_STATUS=timeout" >> $GITHUB_ENV
-          exit 1
+          echo "Run integration test in SIT server..."      
+          echo "INTG_TEST_STATUS=NotRun" >> $GITHUB_ENV
 
       # Send slack notification
       - name: Send slack notification
@@ -185,7 +192,7 @@ jobs:
             {
               "workflow": "${{ github.repository }} | ${{ github.workflow }}",
               "status": "${{ job.status }}",
-              "details": "${{ env.SMOKE_TEST_STATUS }}",
+              "details": "${{ env.INTG_TEST_STATUS }}",
               "ref": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
             }
         env:

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/.github/workflows/smoke-test-cli.yaml

@@ -42,7 +42,7 @@ jobs:
     steps:
 
     - name: Checkout Moonshot (Pull Request Review)
-      if: github.event_name == 'pull_request_review'
+      if: github.event_name == 'pull_request'
       uses: actions/checkout@v4
       with:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
@@ -69,7 +69,7 @@ jobs:
         echo "${{ secrets.ENVIRONMENT_VARS }}" >> .env
 
     - name: Checkout Moonshot Data (Pull Request Review)
-      if: github.event_name == 'pull_request_review'
+      if: github.event_name == 'pull_request'
       uses: actions/checkout@v4
       with: 
          repository: aiverify-foundation/moonshot-data
@@ -91,7 +91,7 @@ jobs:
         pip install -r requirements.txt
 
     - name: Checkout Moonshot UI (Pull Request Review)
-      if: github.event_name == 'pull_request_review'
+      if: github.event_name == 'pull_request'
       uses: actions/checkout@v4
       with:
         repository: aiverify-foundation/moonshot-ui

aiverify_moonshot-0.5.0/.github/workflows/uat-deploy.yaml

@@ -0,0 +1,76 @@
+name: UAT Deploy
+
+on:
+  pull_request:
+    branches:
+      - main
+    types: [closed]
+
+jobs:
+  build-python-package:
+    if: ${{ github.event.pull_request.merged }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          ref: main
+          submodules: recursive
+
+      - name: Setup python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python3 -m pip install --upgrade pip
+          pip install -r requirements.txt
+
+      - name: Bump version
+        run: |
+          echo "Bumping version..."
+          pip install bump2version
+          bump2version patch
+
+      - name: Package test PyPI
+        run: |
+          echo "Packaging test PyPI..."
+          pip install build
+          python3 -m build
+
+      - name: Store the distribution packages
+        uses: actions/upload-artifact@v3
+        with:
+          name: python-package
+          path: dist/
+
+  publish-to-testpypi:
+    needs:
+      - build-python-package
+    runs-on: ubuntu-latest
+
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/aiverify-moonshot
+
+    permissions:
+      id-token: write  # IMPORTANT: mandatory for trusted publishing
+
+    steps:
+      - name: Download python package
+        uses: actions/download-artifact@v3
+        with:
+          name: python-package
+          path: dist/
+
+      - name: Publish to test PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+
+
+
+
+

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/PKG-INFO

@@ -1,15 +1,12 @@
 Metadata-Version: 2.3
 Name: aiverify-moonshot
-Version: 0.4.11
+Version: 0.5.0
 Summary: AI Verify advances Gen AI testing with Project Moonshot.
 Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
 Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
 Project-URL: Issues, https://github.com/aiverify-foundation/moonshot/issues
 Author-email: AI Verify Foundation <info@aiverify.sg>
 License: Apache-2.0
-License-File: AUTHORS.md
-License-File: LICENSE.md
-License-File: NOTICES.md
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
@@ -24,6 +21,7 @@ Requires-Dist: pydantic==2.8.2
 Requires-Dist: pyparsing>=3.1.4
 Requires-Dist: python-dotenv>=1.0.1
 Requires-Dist: python-slugify>=8.0.4
+Requires-Dist: tenacity>=8.5.0
 Requires-Dist: xxhash>=3.5.0
 Provides-Extra: all
 Requires-Dist: cmd2>=2.4.3; extra == 'all'
@@ -46,7 +44,7 @@ Description-Content-Type: text/markdown
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.11**
+**Version 0.5.0**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/README.md

@@ -2,7 +2,7 @@
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.11**
+**Version 0.5.0**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/moonshot/integrations/cli/benchmark/cookbook.py

@@ -256,11 +256,7 @@ def run_cookbook(args) -> None:
         if isinstance(args.random_seed, bool) or not isinstance(args.random_seed, int):
             raise TypeError(ERROR_BENCHMARK_RUN_COOKBOOK_RANDOM_SEED_VALIDATION)
 
-        if (
-            not isinstance(args.system_prompt, str)
-            or not args.system_prompt
-            or args.system_prompt is None
-        ):
+        if not isinstance(args.system_prompt, str) or args.system_prompt is None:
             raise TypeError(ERROR_BENCHMARK_RUN_COOKBOOK_SYS_PROMPT_VALIDATION)
 
         if (

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/moonshot/integrations/cli/benchmark/recipe.py

@@ -337,11 +337,7 @@ def run_recipe(args) -> None:
         if isinstance(args.random_seed, bool) or not isinstance(args.random_seed, int):
             raise TypeError(ERROR_BENCHMARK_RUN_RECIPE_RANDOM_SEED_VALIDATION)
 
-        if (
-            not isinstance(args.system_prompt, str)
-            or not args.system_prompt
-            or args.system_prompt is None
-        ):
+        if not isinstance(args.system_prompt, str) or args.system_prompt is None:
             raise TypeError(ERROR_BENCHMARK_RUN_RECIPE_SYS_PROMPT_VALIDATION)
 
         if (

{aiverify_moonshot-0.4.11 → aiverify_moonshot-0.5.0}/moonshot/integrations/cli/cli_errors.py

@@ -348,6 +348,9 @@ ERROR_COMMON_ADD_ENDPOINT_MAX_CALLS_PER_SECOND_VALIDATION = "The 'max_calls_per_
 ERROR_COMMON_ADD_ENDPOINT_MAX_CONCURRENCY_VALIDATION = (
     "The 'max_concurrency' argument must be a non-empty positive integer and not None."
 )
+ERROR_COMMON_ADD_ENDPOINT_MODEL_VALIDATION = (
+    "The 'model' argument must be a string and not None."
+)
 ERROR_COMMON_ADD_ENDPOINT_PARAMS_VALIDATION = (
     "The 'params' argument must be a string representation of a dictionary."
 )