aiverify-moonshot 0.5.0__tar.gz → 0.5.1__tar.gz

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/.github/workflows/integration-test.yaml

@@ -110,7 +110,7 @@ jobs:
       run: |
         source venv/bin/activate
         pip install nltk
-        python -c "import nltk; nltk.download('stopwords')"
+        python -c "import nltk; nltk.download('stopwords');nltk.download('averaged_perceptron_tagger'); nltk.download('omw');nltk.download('universal_tagset'); nltk.download('wordnet');nltk.download('punkt')"
     
     - name: Setup Moonshot UI
       run: |

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/PKG-INFO

@@ -1,12 +1,15 @@
-Metadata-Version: 2.3
+Metadata-Version: 2.4
 Name: aiverify-moonshot
-Version: 0.5.0
+Version: 0.5.1
 Summary: AI Verify advances Gen AI testing with Project Moonshot.
 Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
 Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
 Project-URL: Issues, https://github.com/aiverify-foundation/moonshot/issues
 Author-email: AI Verify Foundation <info@aiverify.sg>
 License: Apache-2.0
+License-File: AUTHORS.md
+License-File: LICENSE.md
+License-File: NOTICES.md
 Classifier: Development Status :: 3 - Alpha
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Programming Language :: Python :: 3
@@ -26,7 +29,7 @@ Requires-Dist: xxhash>=3.5.0
 Provides-Extra: all
 Requires-Dist: cmd2>=2.4.3; extra == 'all'
 Requires-Dist: dependency-injector>=4.41.0; extra == 'all'
-Requires-Dist: fastapi>=0.112.2; extra == 'all'
+Requires-Dist: fastapi>=0.115.4; extra == 'all'
 Requires-Dist: rich>=13.8.0; extra == 'all'
 Requires-Dist: typing-extensions>=4.12.2; extra == 'all'
 Requires-Dist: uvicorn>=0.30.6; extra == 'all'
@@ -35,7 +38,7 @@ Requires-Dist: cmd2>=2.4.3; extra == 'cli'
 Requires-Dist: rich>=13.8.0; extra == 'cli'
 Provides-Extra: web-api
 Requires-Dist: dependency-injector>=4.41.0; extra == 'web-api'
-Requires-Dist: fastapi>=0.112.2; extra == 'web-api'
+Requires-Dist: fastapi>=0.115.4; extra == 'web-api'
 Requires-Dist: typing-extensions>=4.12.2; extra == 'web-api'
 Requires-Dist: uvicorn>=0.30.6; extra == 'web-api'
 Description-Content-Type: text/markdown
@@ -44,7 +47,7 @@ Description-Content-Type: text/markdown
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.5.0**
+**Version 0.5.1**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/README.md

@@ -2,7 +2,7 @@
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.5.0**
+**Version 0.5.1**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/moonshot/integrations/cli/common/dataset.py

@@ -5,13 +5,12 @@ from rich.console import Console
 from rich.table import Table
 
 from moonshot.api import (
+    api_convert_dataset,
     api_delete_dataset,
+    api_download_dataset,
     api_get_all_datasets,
     api_get_all_datasets_name,
-    api_convert_dataset, 
-    api_download_dataset
 )
-
 from moonshot.integrations.cli.cli_errors import (
     ERROR_BENCHMARK_DELETE_DATASET_DATASET_VALIDATION,
     ERROR_BENCHMARK_LIST_DATASETS_FIND_VALIDATION,
@@ -19,12 +18,12 @@ from moonshot.integrations.cli.cli_errors import (
     ERROR_BENCHMARK_LIST_DATASETS_PAGINATION_VALIDATION_1,
     ERROR_BENCHMARK_VIEW_DATASET_DATASET_FILENAME_VALIDATION,
 )
-
 from moonshot.integrations.cli.common.display_helper import display_view_str_format
 from moonshot.integrations.cli.utils.process_data import filter_data
 
 console = Console()
 
+
 def list_datasets(args) -> list | None:
     """
     List all available datasets.
@@ -155,6 +154,7 @@ def delete_dataset(args) -> None:
     except Exception as e:
         print(f"[delete_dataset]: {str(e)}")
 
+
 def convert_dataset(args) -> None:
     """
     Convert an existing dataset to a new format.
@@ -350,8 +350,7 @@ download_dataset_args.add_argument(
     type=literal_eval,
     help=(
         "Params of the new dataset in dictionary format. For example: \n"
-        "1. For 'csv' method: \"{'csv_file_path': '/path/to/your/file.csv'}\"\n"
-        "2. For 'hf' method: \"{'dataset_name': 'cais_mmlu', 'dataset_config': 'college_biology', 'split': 'test', "
+        "{'dataset_name': 'cais_mmlu', 'dataset_config': 'college_biology', 'split': 'test', "
         "'input_col': ['questions','choices'], 'target_col': 'answer'}\""
     ),
 )

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/moonshot/integrations/web_api/app.py

@@ -71,7 +71,7 @@ def create_app(cfg: providers.Configuration) -> CustomFastAPI:
     }
 
     app: CustomFastAPI = CustomFastAPI(
-        title="Project Moonshot", version="0.5.0", **app_kwargs
+        title="Project Moonshot", version="0.5.1", **app_kwargs
     )
 
     if cfg.cors.enabled():

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/moonshot/integrations/web_api/schemas/cookbook_response_model.py

@@ -8,4 +8,4 @@ from moonshot.src.cookbooks.cookbook_arguments import (
 class CookbookResponseModel(CookbookPydanticModel):
     total_prompt_in_cookbook: Optional[int] = None
     total_dataset_in_cookbook: Optional[int] = None
-    endpoint_required: Optional[list[str]] = None
+    required_config: dict | None = None

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/moonshot/integrations/web_api/schemas/recipe_response_model.py

@@ -5,4 +5,4 @@ from moonshot.src.recipes.recipe_arguments import RecipeArguments as RecipePydan
 
 class RecipeResponseModel(RecipePydanticModel):
     total_prompt_in_recipe: Optional[int] = None
-    endpoint_required: Optional[list[str]] = None
+    required_config: dict | None = None

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/moonshot/integrations/web_api/services/cookbook_service.py

@@ -5,7 +5,10 @@ from .... import api as moonshot_api
 from ..schemas.cookbook_create_dto import CookbookCreateDTO, CookbookUpdateDTO
 from ..schemas.cookbook_response_model import CookbookResponseModel
 from ..services.base_service import BaseService
-from ..services.recipe_service import get_total_prompt_in_recipe, get_endpoint_dependency_in_recipe
+from ..services.recipe_service import (
+    get_metric_dependency_in_recipe,
+    get_total_prompt_in_recipe,
+)
 from ..services.utils.exceptions_handler import exception_handler
 
 
@@ -63,25 +66,28 @@ class CookbookService(BaseService):
                 if cookbook not in cookbooks_list:
                     cookbooks_list.append(cookbook)
                     if count:
-                        cookbook.total_prompt_in_cookbook, cookbook.total_dataset_in_cookbook = (
-                            get_total_prompt_and_dataset_in_cookbook(cookbook)
-                        )
+                        (
+                            cookbook.total_prompt_in_cookbook,
+                            cookbook.total_dataset_in_cookbook,
+                        ) = get_total_prompt_and_dataset_in_cookbook(cookbook)
 
             if tags and cookbooks_recipe_has_tags(tags, cookbook):
                 if cookbook not in cookbooks_list:
                     cookbooks_list.append(cookbook)
                     if count:
-                        cookbook.total_prompt_in_cookbook, cookbook.total_dataset_in_cookbook = (
-                            get_total_prompt_and_dataset_in_cookbook(cookbook)
-                        )
+                        (
+                            cookbook.total_prompt_in_cookbook,
+                            cookbook.total_dataset_in_cookbook,
+                        ) = get_total_prompt_and_dataset_in_cookbook(cookbook)
 
             if categories and cookbooks_recipe_has_categories(categories, cookbook):
                 if cookbook not in cookbooks_list:
                     cookbooks_list.append(cookbook)
                     if count:
-                        cookbook.total_prompt_in_cookbook, cookbook.total_dataset_in_cookbook = (
-                            get_total_prompt_and_dataset_in_cookbook(cookbook)
-                        )
+                        (
+                            cookbook.total_prompt_in_cookbook,
+                            cookbook.total_dataset_in_cookbook,
+                        ) = get_total_prompt_and_dataset_in_cookbook(cookbook)
 
             if categories_excluded and cookbooks_recipe_has_categories(
                 categories_excluded, cookbook
@@ -89,7 +95,7 @@ class CookbookService(BaseService):
                 cookbooks_list.remove(cookbook)
 
         for cookbook in cookbooks_list:
-            cookbook.endpoint_required = cookbook_endpoint_dependency(cookbook)
+            cookbook.required_config = cookbook_metrics_dependency(cookbook)
 
         return cookbooks_list
 
@@ -147,7 +153,9 @@ def get_total_prompt_and_dataset_in_cookbook(cookbook: Cookbook) -> tuple[int, i
         int: The total count of prompts within the cookbook.
     """
     recipes = moonshot_api.api_read_recipes(cookbook.recipes)
-    total_prompts, total_datasets = zip(*(get_total_prompt_in_recipe(Recipe(**recipe)) for recipe in recipes))
+    total_prompts, total_datasets = zip(
+        *(get_total_prompt_in_recipe(Recipe(**recipe)) for recipe in recipes)
+    )
     return sum(total_prompts), sum(total_datasets)
 
 
@@ -197,25 +205,48 @@ def cookbooks_recipe_has_categories(categories: str, cookbook: Cookbook) -> bool
             return True
     return False
 
+
 @staticmethod
-def cookbook_endpoint_dependency(cookbook: Cookbook) -> list[str] | None:
+def cookbook_metrics_dependency(cookbook: Cookbook) -> dict | None:
     """
-    Retrieve a list of endpoint dependencies for all recipes in a given cookbook.
+    Retrieve a list of endpoint and configuration dependencies for all recipes in a given cookbook.
 
     Args:
         cookbook (Cookbook): The cookbook object containing the recipe IDs.
 
     Returns:
-        list[str] | None: A list of endpoint dependencies if any are found, otherwise None.
+        dict[str, list[str]] | None: A dictionary with aggregated endpoint and configuration dependencies
+        if any are found, otherwise None.
     """
     recipes_in_cookbook = cookbook.recipes
     recipes = moonshot_api.api_read_recipes(recipes_in_cookbook)
-    list_of_endpoints = set()
+    aggregated_endpoints = set()
+    aggregated_configurations = {}
 
     for recipe in recipes:
         recipe = Recipe(**recipe)
-        endpoints = get_endpoint_dependency_in_recipe(recipe)
-        if endpoints:
-            list_of_endpoints.update(endpoints)
-
-    return list(list_of_endpoints) if list_of_endpoints else None
+        dependency = get_metric_dependency_in_recipe(recipe)
+        if dependency:
+            # Aggregate endpoints
+            aggregated_endpoints.update(dependency.get("endpoints", []))
+
+            # Aggregate configurations
+            for key, value in dependency.get("configurations", {}).items():
+                if key in aggregated_configurations:
+                    aggregated_configurations[key].extend(
+                        v for v in value if v not in aggregated_configurations[key]
+                    )
+                else:
+                    aggregated_configurations[key] = value
+
+    aggregated_dependencies = {
+        "endpoints": list(aggregated_endpoints),
+        "configurations": aggregated_configurations,
+    }
+
+    return (
+        aggregated_dependencies
+        if aggregated_dependencies["endpoints"]
+        or aggregated_dependencies["configurations"]
+        else None
+    )

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/moonshot/integrations/web_api/services/recipe_service.py

@@ -85,7 +85,7 @@ class RecipeService(BaseService):
                 filtered_recipes.sort(key=lambda x: x.id)
 
         for recipe in filtered_recipes:
-            recipe.endpoint_required = get_endpoint_dependency_in_recipe(recipe)
+            recipe.required_config = get_metric_dependency_in_recipe(recipe)
 
         return filtered_recipes
 
@@ -157,28 +157,49 @@ def get_total_prompt_in_recipe(recipe: Recipe) -> tuple[int, int]:
 
     return total_prompt_count, int(recipe.stats.get("num_of_datasets", 0))
 
+
 @staticmethod
-def get_endpoint_dependency_in_recipe(recipe: Recipe) -> list[str] | None:
+def get_metric_dependency_in_recipe(recipe: Recipe) -> dict | None:
     """
-    Retrieve the list of endpoint dependencies for a given recipe.
+    Retrieve endpoint and configuration dependencies for a given recipe.
 
-    This function fetches all metrics and their associated endpoints, then
-    matches the metrics in the provided recipe to find and compile a list
-    of endpoint dependencies.
+    This function gathers all available metrics along with their endpoints and configurations,
+    then identifies and compiles the dependencies based on the metrics present in the provided recipe.
 
     Args:
-        recipe (Recipe): The recipe object containing the metrics information.
+        recipe (Recipe): The recipe object containing metrics information.
 
     Returns:
-        list[str] | None: A list of endpoint dependencies if found, otherwise None.
+        dict[str, dict[str, list[str]]] | None: A dictionary with 'endpoints' and 'configurations' as keys,
+        where 'endpoints' is a list of endpoint dependencies and 'configurations' is a dictionary of configuration
+        dependencies. Returns None if no dependencies are found.
     """
     metrics = recipe.metrics
     all_metrics = moonshot_api.api_get_all_metric()
-    
-    endpoints = set()
+    aggregated_endpoints = set()
+    aggregated_configurations = {}
+
     for metric in metrics:
-        for m in all_metrics:
-            if m['id'] == metric:
-                endpoints.update(m['endpoints'])
-    
-    return list(endpoints) if endpoints else None
+        metric_data = next((m for m in all_metrics if m["id"] == metric), None)
+        if metric_data:
+            # Aggregate endpoints
+            aggregated_endpoints.update(metric_data.get("endpoints", []))
+
+            # Aggregate configurations
+            for key, value in metric_data.get("configurations", {}).items():
+                if key in aggregated_configurations:
+                    aggregated_configurations[key].extend(
+                        v for v in value if v not in aggregated_configurations[key]
+                    )
+                else:
+                    aggregated_configurations[key] = value
+
+    aggregated_data = {
+        "endpoints": list(aggregated_endpoints),
+        "configurations": aggregated_configurations,
+    }
+    return (
+        aggregated_data
+        if aggregated_data["endpoints"] or aggregated_data["configurations"]
+        else None
+    )

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "aiverify-moonshot"
-version = "0.5.0"
+version = "0.5.1"
 authors = [
     { name="AI Verify Foundation", email="info@aiverify.sg" }
 ]
@@ -35,7 +35,7 @@ dependencies = [
 [project.optional-dependencies]
 web-api = [
     "typing-extensions>=4.12.2",
-    "fastapi>=0.112.2",
+    "fastapi>=0.115.4",
     "uvicorn>=0.30.6",
     "dependency-injector>=4.41.0",
 ]
@@ -97,7 +97,7 @@ allow-direct-references = true
 
 [tool.poetry]
 name = "aiverify-moonshot"
-version = "0.5.0"
+version = "0.5.1"
 description = "A simple and modular tool to evaluate and red-team any LLM application."
 authors = ["The Moonshot Team <our.moonshot.team@gmail.com>"]
 readme = "README.md"
@@ -116,7 +116,7 @@ pandas = ">=2.2.2"
 numpy = ">=1.26.4"
 tenacity = ">=8.5.0"
 typing-extensions = ">=4.12.2"
-fastapi = ">=0.112.2"
+fastapi = ">=0.115.4"
 uvicorn = ">=0.30.6"
 dependency-injector = ">=4.41.0"
 cmd2 = ">=2.4.3"

{aiverify_moonshot-0.5.0 → aiverify_moonshot-0.5.1}/requirements.txt

@@ -12,7 +12,7 @@ colorama~=0.4.6 ; python_version >= "3.11" and python_version < "3.12" and platf
 datasets~=2.21.0 ; python_version >= "3.11" and python_version < "3.12"
 dependency-injector~=4.41.0 ; python_version >= "3.11" and python_version < "3.12"
 dill~=0.3.8 ; python_version >= "3.11" and python_version < "3.12"
-fastapi~=0.112.2 ; python_version >= "3.11" and python_version < "3.12"
+fastapi~=0.115.4 ; python_version >= "3.11" and python_version < "3.12"
 filelock~=3.16.1 ; python_version >= "3.11" and python_version < "3.12"
 frozenlist~=1.4.1 ; python_version >= "3.11" and python_version < "3.12"
 fsspec~=2024.6.1 ; python_version >= "3.11" and python_version < "3.12"
@@ -46,7 +46,7 @@ requests~=2.32.3 ; python_version >= "3.11" and python_version < "3.12"
 rich~=13.9.1 ; python_version >= "3.11" and python_version < "3.12"
 six~=1.16.0 ; python_version >= "3.11" and python_version < "3.12"
 sniffio~=1.3.1 ; python_version >= "3.11" and python_version < "3.12"
-starlette~=0.38.5 ; python_version >= "3.11" and python_version < "3.12"
+starlette~=0.41.2 ; python_version >= "3.11" and python_version < "3.12"
 tenacity~=8.5.0 ; python_version >= "3.11" and python_version < "3.12"
 text-unidecode~=1.3 ; python_version >= "3.11" and python_version < "3.12"
 tqdm~=4.66.5 ; python_version >= "3.11" and python_version < "3.12"