aiverify-moonshot 0.4.10__tar.gz → 0.4.11__tar.gz

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/.github/workflows/smoke-test-cli.yaml

@@ -4,10 +4,10 @@ name: Moonshot CLI Smoke Test
 
 on:
   # Runs on Pull Request Review
-  pull_request_review:
-   types: [submitted]
-   branches:
-     - 'dev_main'
+  pull_request:
+    types: [ assigned, synchronize ]
+    branches:
+      - 'dev_main'
 
   # Run this workflow manually from Actions tab
   workflow_dispatch:
@@ -35,7 +35,7 @@ concurrency:
 
 jobs:
   smoke-test:
-
+    if: (github.event_name == 'pull_request' && github.event.pull_request.assignee != null) || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     timeout-minutes: 20
 

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/.github/workflows/smoke-test.yaml

@@ -3,11 +3,11 @@
 name: Moonshot Smoke Test
 
 on:
-  # Runs on Pull Request Review 
-  pull_request_review:
-   types: [submitted]
+  # Runs when a pull request to main is being assigned
+  pull_request:
+   types: [ assigned, synchronize ]
    branches:
-     - 'dev_main'
+      - 'dev_main'
 
   # Run this workflow manually from Actions tab
   workflow_dispatch:
@@ -35,14 +35,13 @@ concurrency:
 
 jobs:
   smoke-test:
-    if: (github.event_name == 'pull_request_review' && github.event.pull_request.base.ref == 'dev_main') || github.event_name == 'workflow_dispatch'
+    if: (github.event_name == 'pull_request' && github.event.pull_request.assignee != null) || github.event_name == 'workflow_dispatch'
     runs-on: ubuntu-latest
     timeout-minutes: 60
-
+    
     steps:
-
-    - name: Checkout Moonshot (Pull Request Review)
-      if: github.event_name == 'pull_request_review'
+    - name: Checkout Moonshot (Pull Request)
+      if: github.event_name == 'pull_request'
       uses: actions/checkout@v4
       with:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
@@ -67,9 +66,8 @@ jobs:
         pip install -r requirements.txt
         touch .env
         echo "${{ secrets.ENVIRONMENT_VARS }}" >> .env
-
     - name: Checkout Moonshot Data (Pull Request Review)
-      if: github.event_name == 'pull_request_review'
+      if: github.event_name == 'pull_request'
       uses: actions/checkout@v4
       with: 
          repository: aiverify-foundation/moonshot-data
@@ -89,9 +87,8 @@ jobs:
         source venv/bin/activate
         cd moonshot-data
         pip install -r requirements.txt
-
     - name: Checkout Moonshot UI (Pull Request Review)
-      if: github.event_name == 'pull_request_review'
+      if: github.event_name == 'pull_request'
       uses: actions/checkout@v4
       with:
         repository: aiverify-foundation/moonshot-ui
@@ -105,16 +102,22 @@ jobs:
         repository: aiverify-foundation/moonshot-ui
         ref: ${{ inputs.moonshot_ui_branch }}
         path: moonshot-ui
-    
+
+      # Download NLTK stopwords
+    - name: Download NLTK stopwords
+      run: |
+        source venv/bin/activate
+        pip install nltk
+        python -c "import nltk; nltk.download('stopwords')"
+        
     - name: Setup Moonshot UI
       run: |
         cd moonshot-ui
-        npm ci
+        npm install
         npm run build
         cd ../
         source venv/bin/activate
         python -m moonshot web &
-
     - name: Checkout Smoke Test 
       uses: actions/checkout@v4
       with:
@@ -129,7 +132,9 @@ jobs:
         MOONSHOT_URL: ${{ secrets.MOONSHOT_URL }}
         MOONSHOT_PORT_NUMBER: ${{ secrets.MOONSHOT_PORT_NUMBER }}
       run: |
+         source venv/bin/activate
          cd moonshot-smoke-testing
-         npm ci
-         npx playwright install --with-deps
-         AZURE_OPENAI_URI="$AZURE_OPENAI_URI" AZURE_OPENAI_TOKEN="$AZURE_OPENAI_TOKEN" ADDITIONAL_PARAMETERS="$ADDITIONAL_PARAMETERS" MOONSHOT_URL="$MOONSHOT_URL" MOONSHOT_PORT_NUMBER="$MOONSHOT_PORT_NUMBER" npx playwright test tests/smoke-test.spec.ts
+         npm install --global yarn
+         yarn add playwright
+         yarn playwright install --with-deps
+         AZURE_OPENAI_URI="$AZURE_OPENAI_URI" AZURE_OPENAI_TOKEN="$AZURE_OPENAI_TOKEN" ADDITIONAL_PARAMETERS="$ADDITIONAL_PARAMETERS" MOONSHOT_URL="$MOONSHOT_URL" MOONSHOT_PORT_NUMBER="$MOONSHOT_PORT_NUMBER" DEBUG=pw:api npx playwright test tests/smoke-test.spec.ts

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: aiverify-moonshot
-Version: 0.4.10
+Version: 0.4.11
 Summary: AI Verify advances Gen AI testing with Project Moonshot.
 Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
 Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
@@ -15,38 +15,38 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
 Classifier: Programming Language :: Python :: 3.11
 Requires-Python: >=3.11
-Requires-Dist: datasets==2.21.0
-Requires-Dist: ijson==3.3.0
-Requires-Dist: jinja2==3.1.4
-Requires-Dist: numpy==1.26.4
-Requires-Dist: pandas==2.2.2
+Requires-Dist: datasets>=2.21.0
+Requires-Dist: ijson>=3.3.0
+Requires-Dist: jinja2>=3.1.4
+Requires-Dist: numpy>=1.26.4
+Requires-Dist: pandas>=2.2.2
 Requires-Dist: pydantic==2.8.2
-Requires-Dist: pyparsing==3.1.4
-Requires-Dist: python-dotenv==1.0.1
-Requires-Dist: python-slugify==8.0.4
-Requires-Dist: xxhash==3.5.0
+Requires-Dist: pyparsing>=3.1.4
+Requires-Dist: python-dotenv>=1.0.1
+Requires-Dist: python-slugify>=8.0.4
+Requires-Dist: xxhash>=3.5.0
 Provides-Extra: all
-Requires-Dist: cmd2==2.4.3; extra == 'all'
-Requires-Dist: dependency-injector==4.41.0; extra == 'all'
-Requires-Dist: fastapi==0.112.2; extra == 'all'
-Requires-Dist: rich==13.8.0; extra == 'all'
-Requires-Dist: typing-extensions==4.12.2; extra == 'all'
-Requires-Dist: uvicorn==0.30.6; extra == 'all'
+Requires-Dist: cmd2>=2.4.3; extra == 'all'
+Requires-Dist: dependency-injector>=4.41.0; extra == 'all'
+Requires-Dist: fastapi>=0.112.2; extra == 'all'
+Requires-Dist: rich>=13.8.0; extra == 'all'
+Requires-Dist: typing-extensions>=4.12.2; extra == 'all'
+Requires-Dist: uvicorn>=0.30.6; extra == 'all'
 Provides-Extra: cli
-Requires-Dist: cmd2==2.4.3; extra == 'cli'
-Requires-Dist: rich==13.8.0; extra == 'cli'
+Requires-Dist: cmd2>=2.4.3; extra == 'cli'
+Requires-Dist: rich>=13.8.0; extra == 'cli'
 Provides-Extra: web-api
-Requires-Dist: dependency-injector==4.41.0; extra == 'web-api'
-Requires-Dist: fastapi==0.112.2; extra == 'web-api'
-Requires-Dist: typing-extensions==4.12.2; extra == 'web-api'
-Requires-Dist: uvicorn==0.30.6; extra == 'web-api'
+Requires-Dist: dependency-injector>=4.41.0; extra == 'web-api'
+Requires-Dist: fastapi>=0.112.2; extra == 'web-api'
+Requires-Dist: typing-extensions>=4.12.2; extra == 'web-api'
+Requires-Dist: uvicorn>=0.30.6; extra == 'web-api'
 Description-Content-Type: text/markdown
 
 <div align="center">
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.10**
+**Version 0.4.11**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/README.md

@@ -2,7 +2,7 @@
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.10**
+**Version 0.4.11**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/moonshot/integrations/cli/benchmark/cookbook.py

@@ -467,7 +467,10 @@ def _display_view_cookbook(cookbook_info):
     recipes_list = api_read_recipes(recipes)
     if recipes_list:
         table = Table(
-            title="View Cookbook", show_lines=True, expand=True, header_style="bold"
+            title=f'Cookbook "{name}"',
+            show_lines=True,
+            expand=True,
+            header_style="bold",
         )
         table.add_column("No.", width=2)
         table.add_column("Recipe", justify="left", width=78)
@@ -482,7 +485,6 @@ def _display_view_cookbook(cookbook_info):
                 datasets,
                 prompt_templates,
                 metrics,
-                attack_strategies,
                 grading_scale,
                 stats,
             ) = recipe.values()
@@ -494,9 +496,6 @@ def _display_view_cookbook(cookbook_info):
                 "Prompt Templates", prompt_templates
             )
             metrics_info = display_view_list_format("Metrics", metrics)
-            attack_strategies_info = display_view_list_format(
-                "Attack Strategies", attack_strategies
-            )
             grading_scale_info = _display_view_grading_scale_format(
                 "Grading Scale", grading_scale
             )
@@ -506,7 +505,9 @@ def _display_view_cookbook(cookbook_info):
                 f"[red]id: {id}[/red]\n\n[blue]{name}[/blue]\n{description}\n\n"
                 f"{tags_info}\n\n{categories_info}\n\n{grading_scale_info}\n\n{stats_info}"
             )
-            contains_info = f"{datasets_info}\n\n{prompt_templates_info}\n\n{metrics_info}\n\n{attack_strategies_info}"
+            contains_info = (
+                f"{datasets_info}\n\n{prompt_templates_info}\n\n{metrics_info}"
+            )
 
             table.add_section()
             table.add_row(str(recipe_id), recipe_info, contains_info)

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/moonshot/integrations/web_api/app.py

@@ -71,7 +71,7 @@ def create_app(cfg: providers.Configuration) -> CustomFastAPI:
     }
 
     app: CustomFastAPI = CustomFastAPI(
-        title="Project Moonshot", version="0.4.10", **app_kwargs
+        title="Project Moonshot", version="0.4.11", **app_kwargs
     )
 
     if cfg.cors.enabled():

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/moonshot/src/bookmark/bookmark.py

@@ -11,9 +11,9 @@ from moonshot.src.messages_constants import (
     BOOKMARK_ADD_BOOKMARK_VALIDATION_ERROR,
     BOOKMARK_DELETE_ALL_BOOKMARK_ERROR,
     BOOKMARK_DELETE_ALL_BOOKMARK_SUCCESS,
-    BOOKMARK_DELETE_BOOKMARK_FAIL,
     BOOKMARK_DELETE_BOOKMARK_ERROR,
     BOOKMARK_DELETE_BOOKMARK_ERROR_1,
+    BOOKMARK_DELETE_BOOKMARK_FAIL,
     BOOKMARK_DELETE_BOOKMARK_SUCCESS,
     BOOKMARK_EXPORT_BOOKMARK_ERROR,
     BOOKMARK_EXPORT_BOOKMARK_VALIDATION_ERROR,
@@ -186,7 +186,9 @@ class Bookmark:
             if (
                 bookmark_info is not None
                 and isinstance(bookmark_info, tuple)
-                and all(isinstance(item, str) for item in bookmark_info)
+                and all(
+                    isinstance(item, str) for item in bookmark_info[1:]
+                )  # Check if the rest are strings besides id
             ):
                 return BookmarkArguments.from_tuple_to_dict(bookmark_info)
             else:
@@ -210,9 +212,11 @@ class Bookmark:
         """
         if isinstance(bookmark_name, str) and bookmark_name:
             try:
-
                 bookmark_info = Storage.read_database_record(
-                self.db_instance, (bookmark_name,), Bookmark.sql_select_bookmark_record)
+                    self.db_instance,
+                    (bookmark_name,),
+                    Bookmark.sql_select_bookmark_record,
+                )
                 if bookmark_info is not None:
                     sql_delete_bookmark_record = textwrap.dedent(
                         f"""
@@ -222,7 +226,10 @@ class Bookmark:
                     Storage.delete_database_record_in_table(
                         self.db_instance, sql_delete_bookmark_record
                     )
-                    return {"success": True, "message": BOOKMARK_DELETE_BOOKMARK_SUCCESS}
+                    return {
+                        "success": True,
+                        "message": BOOKMARK_DELETE_BOOKMARK_SUCCESS,
+                    }
                 else:
                     return {"success": False, "message": BOOKMARK_DELETE_BOOKMARK_FAIL}
             except Exception as e:

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/moonshot/src/connectors/connector.py

@@ -10,6 +10,7 @@ from typing import Callable
 
 from moonshot.src.configs.env_variables import EnvVariables
 from moonshot.src.connectors.connector_prompt_arguments import ConnectorPromptArguments
+from moonshot.src.connectors.connector_response import ConnectorResponse
 from moonshot.src.connectors_endpoints.connector_endpoint_arguments import (
     ConnectorEndpointArguments,
 )
@@ -168,9 +169,9 @@ class Connector:
         return wrapper
 
     @abstractmethod
-    async def get_response(self, prompt: str) -> str:
+    async def get_response(self, prompt: str) -> ConnectorResponse:
         """
-        Abstract method to be implemented by subclasses to get a response from the connector.
+        Abstract method to be implemented by subclasses to obtain a response from the connector.
 
         This method should asynchronously send a prompt to the connector's API and return the response.
 
@@ -178,7 +179,8 @@ class Connector:
             prompt (str): The input prompt to be sent to the connector.
 
         Returns:
-            str: The response received from the connector.
+            ConnectorResponse: An instance containing the response received from the connector and any
+            additional metadata.
         """
         pass
 
@@ -277,7 +279,7 @@ class Connector:
         object, which is used to generate the prediction. The method also optionally takes a `prompt_callback` function,
         which is called after the prediction is generated.
 
-        The method first prints a message indicating that it is predicting the prompt. It then records the start time
+        The method logs a message indicating that it is predicting the prompt. It then records the start time
         and uses the `connector` to generate a prediction for the `generated_prompt`. The duration of the prediction
         is calculated and stored in the `generated_prompt`.
 

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/moonshot/src/connectors/connector_prompt_arguments.py

@@ -4,14 +4,21 @@ from typing import Any
 
 from pydantic import BaseModel
 
+from moonshot.src.connectors.connector_response import ConnectorResponse
+
 
 class ConnectorPromptArguments(BaseModel):
+    class Config:
+        arbitrary_types_allowed = True
+
     prompt_index: int  # The index of the prompt in the dataset
 
     prompt: str  # The actual prompt text
 
     target: Any  # The target response for the prompt
 
-    predicted_results: Any = ""  # The predicted results, default is an empty string
+    predicted_results: ConnectorResponse | None = (
+        None  # The predicted results, default is None
+    )
 
     duration: float = 0.0  # The duration it took to get the results, default is 0.0

aiverify_moonshot-0.4.11/moonshot/src/connectors/connector_response.py

@@ -0,0 +1,20 @@
+class ConnectorResponse:
+    def __init__(self, response: str, context: list = []):
+        """
+        Initializes a ConnectorResponse instance.
+
+        Args:
+            response (str): The response text from the connector.
+            context (list, optional): Additional context or metadata related to the response. Defaults to an empty list.
+        """
+        self.response = response
+        self.context = context
+
+    def to_dict(self) -> dict:
+        """
+        Converts the ConnectorResponse instance to a dictionary.
+
+        Returns:
+            dict: A dictionary representation of the ConnectorResponse instance.
+        """
+        return {"response": self.response, "context": self.context}

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/moonshot/src/redteaming/attack/attack_module.py

@@ -31,6 +31,22 @@ class AttackModule:
         INSERT INTO {} (connection_id,context_strategy,prompt_template,attack_module,
         metric,prompt,prepared_prompt,system_prompt,predicted_result,duration,prompt_time)VALUES(?,?,?,?,?,?,?,?,?,?,?)
         """
+    sql_create_chat_history_table = """
+        CREATE TABLE IF NOT EXISTS {} (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        connection_id text NOT NULL,
+        context_strategy text,
+        prompt_template text,
+        attack_module text,
+        metric text,
+        prompt text NOT NULL,
+        prepared_prompt text NOT NULL,
+        system_prompt text,
+        predicted_result text NOT NULL,
+        duration text NOT NULL,
+        prompt_time text NOT NULL
+        );
+    """
 
     def __init__(self, am_id: str, am_arguments: AttackModuleArguments | None = None):
         self.id = am_id
@@ -261,6 +277,15 @@ class AttackModule:
             list: A list of consolidated responses from the specified LLM connector.
         """
         consolidated_responses = []
+
+        # perform a check to see if the target endpoint has its table in the db. if not, create one
+        endpoint_id = target_llm_connector.id.replace("-", "_")
+        if not Storage.check_database_table_exists(self.db_instance, endpoint_id):
+            Storage.create_database_table(
+                self.db_instance,
+                AttackModule.sql_create_chat_history_table.format(endpoint_id),
+            )
+
         for prepared_prompt in list_of_prompts:
             if self.cancel_event.is_set():
                 logger.warning(
@@ -316,6 +341,7 @@ class AttackModule:
             chat_record_tuple (tuple): A tuple containing the chat record information.
             chat_record_id (str): The ID of the chat record.
         """
+
         endpoint_id = chat_record_id.replace("-", "_")
         Storage.create_database_record(
             self.db_instance,
@@ -623,7 +649,8 @@ class RedTeamingPromptArguments(BaseModel):
 
         This method collects all the attributes of the RedTeamingPromptArguments instance and forms a tuple
         with the attribute values in this specific order: conn_id, cs_id, pt_id, am_id, me_id, original_prompt,
-        connector_prompt.prompt, connector_prompt.predicted_results, connector_prompt.duration, start_time.
+        connector_prompt.prompt, system_prompt, connector_prompt.predicted_results.response, 
+        connector_prompt.duration, start_time.
 
         Returns:
             tuple: A tuple representation of the RedTeamingPromptArguments instance.
@@ -637,21 +664,21 @@ class RedTeamingPromptArguments(BaseModel):
             self.original_prompt,
             self.connector_prompt.prompt,
             self.system_prompt,
-            str(self.connector_prompt.predicted_results),
+            self.connector_prompt.predicted_results.response if self.connector_prompt.predicted_results else "",
             str(self.connector_prompt.duration),
             self.start_time,
         )
 
     def to_dict(self) -> dict:
         """
-        Converts the RedTeamingPromptArguments instance into a dict.
+        Converts the RedTeamingPromptArguments instance into a dictionary.
 
-        This method collects all the attributes of the RedTeamingPromptArguments instance and forms a dict
-        with the keys: conn_id, cs_id, pt_id, am_id, me_id, original_prompt, prepared_prompt, system_prompt
+        This method collects all the attributes of the RedTeamingPromptArguments instance and forms a dictionary
+        with the keys: conn_id, cs_id, pt_id, am_id, me_id, original_prompt, system_prompt, prepared_prompt,
         response, duration, start_time.
 
         Returns:
-            dict: A dict representation of the RedTeamingPromptArguments instance.
+            dict: A dictionary representation of the RedTeamingPromptArguments instance.
         """
         return {
             "conn_id": self.conn_id,
@@ -662,7 +689,7 @@ class RedTeamingPromptArguments(BaseModel):
             "original_prompt": self.original_prompt,
             "prepared_prompt": self.connector_prompt.prompt,
             "system_prompt": self.system_prompt,
-            "response": str(self.connector_prompt.predicted_results),
+            "response": self.connector_prompt.predicted_results.response if self.connector_prompt.predicted_results else "",
             "duration": str(self.connector_prompt.duration),
             "start_time": self.start_time,
         }

{aiverify_moonshot-0.4.10 → aiverify_moonshot-0.4.11}/moonshot/src/runs/run_arguments.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import ast
+import json
 
 from pydantic import BaseModel
 
@@ -83,20 +84,7 @@ class RunArguments(BaseModel):
         Returns:
             tuple: A tuple containing the run arguments ready for database insertion.
         """
-        return (
-            self.runner_id,
-            self.runner_type.name.lower(),
-            str(self.runner_args),
-            str(self.endpoints),
-            self.results_file,
-            self.start_time,
-            self.end_time,
-            self.duration,
-            str(self.error_messages),
-            str(self.raw_results),
-            str(self.results),
-            self.status.name.lower(),
-        )
+        return self._to_tuple(include_run_id=False)
 
     def to_tuple(self) -> tuple:
         """
@@ -112,7 +100,42 @@ class RunArguments(BaseModel):
         Returns:
             tuple: A tuple containing the serialized attributes of the RunArguments instance.
         """
-        return (
+        return self._to_tuple(include_run_id=True)
+
+    def _to_tuple(self, include_run_id: bool) -> tuple:
+        """
+        Converts the RunArguments object to a tuple format.
+
+        This method serializes the RunArguments instance into a tuple, including or excluding the run_id based on the
+        include_run_id parameter. It ensures that all dictionary keys are converted to strings and JSON-encodes
+        the raw_results and results attributes.
+
+        Args:
+            include_run_id (bool): A flag indicating whether to include the run_id in the resulting tuple.
+
+        Returns:
+            tuple: A tuple containing the serialized attributes of the RunArguments instance.
+        """
+
+        def convert_keys_to_str(d):
+            """
+            Recursively converts all keys in a dictionary to strings.
+
+            Args:
+                d (dict): The dictionary whose keys need to be converted.
+
+            Returns:
+                dict: A new dictionary with all keys converted to strings.
+            """
+
+            def convert_value(v):
+                if isinstance(v, dict):
+                    return {str(k): convert_value(val) for k, val in v.items()}
+                return v
+
+            return {str(k): convert_value(v) for k, v in d.items()}
+
+        base_tuple = (
             self.runner_id,
             self.runner_type.name.lower(),
             str(self.runner_args),
@@ -122,11 +145,11 @@ class RunArguments(BaseModel):
             self.end_time,
             self.duration,
             str(self.error_messages),
-            str(self.raw_results),
-            str(self.results),
+            json.dumps(convert_keys_to_str(self.raw_results)),
+            json.dumps(self.results),
             self.status.name.lower(),
-            self.run_id,
         )
+        return base_tuple + (self.run_id,) if include_run_id else base_tuple
 
     @classmethod
     def from_tuple(cls, run_record: tuple) -> RunArguments:
@@ -156,7 +179,7 @@ class RunArguments(BaseModel):
             end_time=float(run_record[7]),
             duration=run_record[8],
             error_messages=ast.literal_eval(run_record[9]),
-            raw_results=ast.literal_eval(run_record[10]),
-            results=ast.literal_eval(run_record[11]),
+            raw_results=json.loads(run_record[10]),
+            results=json.loads(run_record[11]),
             status=RunStatus(run_record[12]),
         )