aiverify-moonshot 0.5.1__py3-none-any.whl → 0.6.1__py3-none-any.whl

{aiverify_moonshot-0.5.1.dist-info → aiverify_moonshot-0.6.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: aiverify-moonshot
-Version: 0.5.1
+Version: 0.6.1
 Summary: AI Verify advances Gen AI testing with Project Moonshot.
 Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
 Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
@@ -47,7 +47,7 @@ Description-Content-Type: text/markdown
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.5.1**
+**Version 0.6.1**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.5.1.dist-info → aiverify_moonshot-0.6.1.dist-info}/RECORD

@@ -6,14 +6,14 @@ moonshot/integrations/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJW
 moonshot/integrations/cli/__main__.py,sha256=0VnYSj2AayvDCZ3uXpldPcjMHt2Yd7BWojWzFOGSSl4,679
 moonshot/integrations/cli/active_session_cfg.py,sha256=n8hOFxFjvz26qbEFY4q7iPUZYrGLoeCmXJxmOb_xWUE,20
 moonshot/integrations/cli/cli.py,sha256=9tnzcxcSOjblxCUpyh3pK0ke0bLs3s-63OxXtYoZI2g,2769
-moonshot/integrations/cli/cli_errors.py,sha256=jJ15ngnVJjSByamtOE2Cx79k1mOJnAl36NbCjpPppeM,23672
+moonshot/integrations/cli/cli_errors.py,sha256=ltQKnj9bawpVogQFOgLHbxL_9CcFQf3XOR6yQtdqGS4,24030
 moonshot/integrations/cli/benchmark/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 moonshot/integrations/cli/benchmark/benchmark.py,sha256=QUxr6DU11-XeH6Y3j1uPsZsotshgy64G_cWNf0Rn2_U,6303
-moonshot/integrations/cli/benchmark/cookbook.py,sha256=rT4gK88LhATFxsAxWWYExtPVT09oWPHJtJ1HQkLT3yU,29444
+moonshot/integrations/cli/benchmark/cookbook.py,sha256=_ShWLEb1R_G3zKPmRVbeykRQ5pS_fv4IwbVeLEqYYEM,30265
 moonshot/integrations/cli/benchmark/datasets.py,sha256=Uq5XMNWUp775sz9jCZUZHHmkumPFI7cHVRueHgWm70Q,8965
 moonshot/integrations/cli/benchmark/metrics.py,sha256=SHs-hIa4CIPyOJtxK2U4D6IRHy3ZNsRtZlAMGvF9Qxw,8310
-moonshot/integrations/cli/benchmark/recipe.py,sha256=uAYTBu1EwcCBdf9Iut6eboZ6nBt5_hDwBhru50mZer0,32407
-moonshot/integrations/cli/benchmark/result.py,sha256=TGlIuDkPB1Cfu1u2uyDsWtgQxTOoNdGDzc0vCnGC1zk,11152
+moonshot/integrations/cli/benchmark/recipe.py,sha256=KW0h1Ynga_2yc-jOd-ULQSbO9R7zETOz3qn3T23bqh8,32920
+moonshot/integrations/cli/benchmark/result.py,sha256=o6_yca1PqFtbjWgnUpK8v_y2z6zuxmB_ue6MXEevpAo,11223
 moonshot/integrations/cli/benchmark/run.py,sha256=HBztvG_Zkg1ZAWsFv0QDE43FaEmx92vTWc4h1U3VesU,7438
 moonshot/integrations/cli/benchmark/runner.py,sha256=Y4Vt6Qqn9QzsM6eLUM9m2_XKkW3ctu-2jMTSei_TDPU,7098
 moonshot/integrations/cli/common/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -34,7 +34,7 @@ moonshot/integrations/cli/utils/process_data.py,sha256=QVL5vp2_8ZgGicmCAdeYEHkeb
 moonshot/integrations/web_api/.env.dev,sha256=0z5_Ut8rF-UqFZtgjkH2qoqORhD5_nSs2w_OeX2SteI,182
 moonshot/integrations/web_api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 moonshot/integrations/web_api/__main__.py,sha256=MdnLi_ZF-olAAEJwTPU1iGYFYwo-fNWNT2qfchkH3y4,2050
-moonshot/integrations/web_api/app.py,sha256=14_CWTSuLNyX3zH_vaqFUa4fugOUcOxLqx-IEv0-v34,3651
+moonshot/integrations/web_api/app.py,sha256=Jr6mYvfjiPKMUWU58QxvYS-bpvkUotd728t6up3ZS-w,3651
 moonshot/integrations/web_api/container.py,sha256=DVkJG_qm7ItcG6tgMYOqIj07wpKhPWOOfy6-bEv72y4,5915
 moonshot/integrations/web_api/logging_conf.py,sha256=t3EGRV6tZhV732KXe8_Tiy0fiwVAWxZX5Tt8VTgrrfg,3388
 moonshot/integrations/web_api/log/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -53,11 +53,11 @@ moonshot/integrations/web_api/routes/recipe.py,sha256=WOcq4bm2LP87ovO4Op6cDbUPJ2
 moonshot/integrations/web_api/routes/redteam.py,sha256=t-jNot5_PkV6f5_WBorp1HL437NY5RZzxSE-2NfG0es,24541
 moonshot/integrations/web_api/routes/runner.py,sha256=NQdAmVIOnNgSESX3am6wAE0YLIxHYXlnQbh00_7-SD4,8438
 moonshot/integrations/web_api/schemas/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-moonshot/integrations/web_api/schemas/benchmark_runner_dto.py,sha256=nfNMt_9Xg0YAL5f93dZamu7DxSLvAsz8-tdA_DTCXQQ,322
+moonshot/integrations/web_api/schemas/benchmark_runner_dto.py,sha256=IIn6KeMcwxTSlwXuCnOUhd3x24ucq-goV2brU1OvxT4,369
 moonshot/integrations/web_api/schemas/bookmark_create_dto.py,sha256=C78vG8UG02N7Cmt6RSuS8e4sX_G-MLCiAWT-cF5BE8s,374
-moonshot/integrations/web_api/schemas/cookbook_create_dto.py,sha256=00SPVw7lEpfY9yOFdt1XkvvNAzfFRd7d7CA90qguhuQ,670
+moonshot/integrations/web_api/schemas/cookbook_create_dto.py,sha256=wXC0tu1Q8SpSI3Qk0xKPj1vKsOJEYmfPgU4rl6QopUY,826
 moonshot/integrations/web_api/schemas/cookbook_response_model.py,sha256=COLvaE4Hrz_w-C_HQkB7feztweIr0wkY9h8N6NKNIr8,332
-moonshot/integrations/web_api/schemas/dataset_create_dto.py,sha256=Jr_EbIgqR7K__LQXpXSocfYLE6oN7pGRFzWcDVcFaus,963
+moonshot/integrations/web_api/schemas/dataset_create_dto.py,sha256=GRqIIlQZEpzzEXwAFcbDlxOuKg0JZ399axBjg34LMp8,915
 moonshot/integrations/web_api/schemas/dataset_response_dto.py,sha256=s5x4-UXEWccWhK42E0FPXiHG6VqjuFuph-2t5atEkg4,171
 moonshot/integrations/web_api/schemas/endpoint_create_dto.py,sha256=WS8AfRybrweoOgZx6K6jiNy1Z6J3IZS1PUNnrRxGKyM,678
 moonshot/integrations/web_api/schemas/endpoint_response_model.py,sha256=OmmM2uaPSgB2aqPFfkhseKkI5OKCKilXR19gDmwFlLc,321
@@ -74,12 +74,12 @@ moonshot/integrations/web_api/services/auto_red_team_test_manager.py,sha256=a_aB
 moonshot/integrations/web_api/services/auto_red_team_test_state.py,sha256=GRmvdYLwQdE8gGkYD9Sd4n__yEBajl2pRA_V0J2YObE,1952
 moonshot/integrations/web_api/services/base_service.py,sha256=_MaQEuBpRNNHXNPylZUGaUVCSA5a2jHi9NoKBpvIprs,172
 moonshot/integrations/web_api/services/benchmark_result_service.py,sha256=-oPvLL7b-pEAOeY0gwlngpgImklkUiwvPE6IJo83a7M,909
-moonshot/integrations/web_api/services/benchmark_test_manager.py,sha256=zsB8zTDUvH-hT1c-rmyh71uO9ZuIxYbUP3msh8Hdkm4,4024
+moonshot/integrations/web_api/services/benchmark_test_manager.py,sha256=aPoB6hOfOYqsDliiIzZ0y6cCI0mPDXLK21j9fHXm10U,4076
 moonshot/integrations/web_api/services/benchmark_test_state.py,sha256=MyhTxpAhhP66JF0ua1SMc_IIeIjDxQY5swOXv9cmYaY,1887
 moonshot/integrations/web_api/services/benchmarking_service.py,sha256=lJZeNTqxEPBLrZNX3Z9JIilgwetywSkv0deQkcb8mQs,1257
 moonshot/integrations/web_api/services/bookmark_service.py,sha256=jI9nXs1hjzO0CLG2LKaXSzDApLThkfCvPUkaNNV9A5A,3546
 moonshot/integrations/web_api/services/context_strategy_service.py,sha256=6YKnnG8JlE_1nlnr4Hq7rgz-sxI6oQglK0STaWPFQxQ,710
-moonshot/integrations/web_api/services/cookbook_service.py,sha256=AU2J_YCLIrRlufPgboZ27f89rVVR1zJHXXHwusf_ipc,8877
+moonshot/integrations/web_api/services/cookbook_service.py,sha256=37iJZn4ybe9tugBWB99g1SAN1YUtkmaq2mLQWj_HBQo,8736
 moonshot/integrations/web_api/services/dataset_service.py,sha256=ZWb3FqyDkA0C9qhlQ3X_zR0ohAlwlLsJi-mgKLvXpnI,2407
 moonshot/integrations/web_api/services/endpoint_service.py,sha256=N5SXNAh44UNeBpMhA9baL0VZoTx4sHzpy4y7-Ch8O4E,2395
 moonshot/integrations/web_api/services/metric_service.py,sha256=xWC5Dk8aiU7tuHsxYedTTrEkbA3Ug1pV2nbaBas6cAg,456
@@ -94,7 +94,7 @@ moonshot/integrations/web_api/status_updater/moonshot_ui_webhook.py,sha256=ToyyC
 moonshot/integrations/web_api/status_updater/interface/benchmark_progress_callback.py,sha256=MOs_1CKpNh2m3JUAEoJfmZOBivk80DNtSnRuTCJgzJ4,350
 moonshot/integrations/web_api/status_updater/interface/redteam_progress_callback.py,sha256=JRczi3vCq6oPfOddPrF4OCdyHQYAVxgPWK-qOJxElKg,350
 moonshot/integrations/web_api/temp/.gitkeep,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-moonshot/integrations/web_api/types/types.py,sha256=AN0Xf61lx2c5AFAYoXA8mVL5iufVBpwYlIPdo8gv-ls,2395
+moonshot/integrations/web_api/types/types.py,sha256=zy1Jlqgju_F7Gb4SaJf7O70egH98D0b4DDCZ384HG2E,2408
 moonshot/src/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 moonshot/src/messages_constants.py,sha256=usbvwitgRdOVY0ARdBbh9uiLNUb6WCdCuSoa64tVMhM,6936
 moonshot/src/api/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -102,7 +102,7 @@ moonshot/src/api/api_bookmark.py,sha256=gg7uJU7ixZtmJ99S3xdgQgcznkY8SjoJaTBkSsno
 moonshot/src/api/api_connector.py,sha256=Q_of-aHPuWkbefMJq4uXctJl89G2Tt6J_HfSuf1hE6g,2234
 moonshot/src/api/api_connector_endpoint.py,sha256=lwfhlWNBJ6QotqffmURtjRmxfzbBlSIAZupeSpMt9VU,5584
 moonshot/src/api/api_context_strategy.py,sha256=uRIfNjKJ_Wk9nSrvbPRfrdQLpG0K6kH9rl5tmmHui40,2151
-moonshot/src/api/api_cookbook.py,sha256=KzGCcJjpAs92FZBloojLbmi5nqK3hd_zX8m5jYWKMMM,5715
+moonshot/src/api/api_cookbook.py,sha256=V05abHvzElrO7LkSyhOMcAHEfCfIgopd6L0cSSO3Dro,6722
 moonshot/src/api/api_dataset.py,sha256=i2KwnZ-6fTm_tyn8cRw8iesrGi7_Nh0-1bFuN7m0TVo,4066
 moonshot/src/api/api_environment_variables.py,sha256=wRx6rm95ItyL_uKUAYfSjcPZNbRxKl1GGS4PpWcTE1s,712
 moonshot/src/api/api_metrics.py,sha256=x5DiysTYQsMmcAS2y2XpgvrPobZk7GT2rhO-MaIRun4,1603
@@ -125,8 +125,8 @@ moonshot/src/connectors_endpoints/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeR
 moonshot/src/connectors_endpoints/connector_endpoint.py,sha256=3U4030gKhvDt4e7VD3lac2FG0kxGJ0DSLPTAUKYOPEk,10018
 moonshot/src/connectors_endpoints/connector_endpoint_arguments.py,sha256=0v9mUHki4l22CK8o8UjATAsFDza9Lutbh4QplLiDXs8,2434
 moonshot/src/cookbooks/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-moonshot/src/cookbooks/cookbook.py,sha256=LjmbJtFwIPhupQlx6LjjIQLMfpwcWZKurduS8MBgqHI,9661
-moonshot/src/cookbooks/cookbook_arguments.py,sha256=iVp3x1-wi_RL7peTSr5TWbXnjuB_w7Uj29kJrQg48Y0,1210
+moonshot/src/cookbooks/cookbook.py,sha256=DdZwRGx5-xTDIKcXtZRpp7Qb9Mm9dNGwXWLQXoQrBBo,10412
+moonshot/src/cookbooks/cookbook_arguments.py,sha256=SmNG8D5qN2K2dcImDaSBPHsna0Gy60ZR49_eTKEsvVU,1445
 moonshot/src/datasets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 moonshot/src/datasets/dataset.py,sha256=-_uhjR7zi50nkLu1WWlPCCWr14VwFUDfhTeeBHOhb70,14236
 moonshot/src/datasets/dataset_arguments.py,sha256=rUcxxo2WTcHhLLV-WoixjOfT_Ju7hFCq811_ctjegt8,1751
@@ -154,7 +154,7 @@ moonshot/src/results/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hS
 moonshot/src/results/result.py,sha256=o56SdhYH-XVfpeeKhN495dJPkU035MmTjRUx48q53lo,4527
 moonshot/src/results/result_arguments.py,sha256=mTR7yajY72PFglfAaa1ajJfvYNV4IBGLXS4VaD53-8c,1334
 moonshot/src/runners/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-moonshot/src/runners/runner.py,sha256=nnYiMHZUSu35jzHoeiEqADhc7iHLKnViyUbd_Qg8WZs,21203
+moonshot/src/runners/runner.py,sha256=7xp4GwiN4fO-ib7CahkaYXoVBB40cuVNJd5DxJxrYsk,21362
 moonshot/src/runners/runner_arguments.py,sha256=Bg4OPSmgr9jZKNAwPH0T3epEHw-6qGrflszFc6oMyEU,1640
 moonshot/src/runners/runner_type.py,sha256=jOfnAnaCYp-rPTRJXhM8hin_dinlR0sMwmimQXvLcJ0,100
 moonshot/src/runs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -172,9 +172,9 @@ moonshot/src/utils/import_modules.py,sha256=T9zTN59PFnvY2rjyWhSV9KSIAHxWV1pyBemF
 moonshot/src/utils/log.py,sha256=YNgD7Eh2OT36XlmVBKCGUTAh9TRp4Akfe4kDdvHASgs,2502
 moonshot/src/utils/pagination.py,sha256=5seymyRoqyENIhKllAatr1T91kMCGFslcvRnJHyMSvc,814
 moonshot/src/utils/timeit.py,sha256=TvuF0w8KWhp0oZFY0cUU3UY0xlGKjchb0OkfYfgVTlc,866
-aiverify_moonshot-0.5.1.dist-info/METADATA,sha256=EtBhJ0bmVoTktwS_p6OUI9JMgY6-gjmhUWfGK3Em7Vk,12419
-aiverify_moonshot-0.5.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-aiverify_moonshot-0.5.1.dist-info/licenses/AUTHORS.md,sha256=mmAbe3i3sT8JZHJMBhxp3i1xRehV0g7WB4T_eyIBuBs,59
-aiverify_moonshot-0.5.1.dist-info/licenses/LICENSE.md,sha256=53izDRmJZZCjpYGfyLqlxnGQN-aNWBxasuzuMXC5Ias,11347
-aiverify_moonshot-0.5.1.dist-info/licenses/NOTICES.md,sha256=vS1zZYAnGjCJdwQ13xv3b2zc30wOS98ZnCKluT-AhHs,123266
-aiverify_moonshot-0.5.1.dist-info/RECORD,,
+aiverify_moonshot-0.6.1.dist-info/METADATA,sha256=Um1dy4p7R1ZqYm9X_wnmzsVi2qclr6trbA11ijKYiRs,12419
+aiverify_moonshot-0.6.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+aiverify_moonshot-0.6.1.dist-info/licenses/AUTHORS.md,sha256=mmAbe3i3sT8JZHJMBhxp3i1xRehV0g7WB4T_eyIBuBs,59
+aiverify_moonshot-0.6.1.dist-info/licenses/LICENSE.md,sha256=53izDRmJZZCjpYGfyLqlxnGQN-aNWBxasuzuMXC5Ias,11347
+aiverify_moonshot-0.6.1.dist-info/licenses/NOTICES.md,sha256=vS1zZYAnGjCJdwQ13xv3b2zc30wOS98ZnCKluT-AhHs,123266
+aiverify_moonshot-0.6.1.dist-info/RECORD,,

moonshot/integrations/cli/benchmark/cookbook.py

@@ -37,7 +37,8 @@ from moonshot.integrations.cli.cli_errors import (
     ERROR_BENCHMARK_RUN_COOKBOOK_ENDPOINTS_VALIDATION_1,
     ERROR_BENCHMARK_RUN_COOKBOOK_NAME_VALIDATION,
     ERROR_BENCHMARK_RUN_COOKBOOK_NO_RESULT,
-    ERROR_BENCHMARK_RUN_COOKBOOK_NUM_OF_PROMPTS_VALIDATION,
+    ERROR_BENCHMARK_RUN_COOKBOOK_PROMPT_SELECTION_PERCENTAGE_RANGE_VALIDATION,
+    ERROR_BENCHMARK_RUN_COOKBOOK_PROMPT_SELECTION_PERCENTAGE_VALIDATION,
     ERROR_BENCHMARK_RUN_COOKBOOK_RANDOM_SEED_VALIDATION,
     ERROR_BENCHMARK_RUN_COOKBOOK_RESULT_PROC_MOD_VALIDATION,
     ERROR_BENCHMARK_RUN_COOKBOOK_RUNNER_PROC_MOD_VALIDATION,
@@ -212,11 +213,12 @@ def run_cookbook(args) -> None:
     The cookbooks are run against the specified endpoints, and the results are processed and displayed.
 
     Args:
-        args: A namespace object from argparse. It should have the following attributes:
+        args (argparse.Namespace): The arguments provided to the command line interface.
+        Expected keys are:
             name (str): The name of the cookbook runner.
             cookbooks (str): A string representation of a list of cookbooks to run.
             endpoints (str): A string representation of a list of endpoints to run.
-            num_of_prompts (int): The number of prompts to run.
+            prompt_selection_percentage (int): The percentage of prompts to run.
             random_seed (int): The random seed number for reproducibility.
             system_prompt (str): The system prompt to use.
             runner_proc_module (str): The runner processing module to use.
@@ -248,10 +250,19 @@ def run_cookbook(args) -> None:
         ):
             raise TypeError(ERROR_BENCHMARK_RUN_COOKBOOK_ENDPOINTS_VALIDATION)
 
-        if isinstance(args.num_of_prompts, bool) or not isinstance(
-            args.num_of_prompts, int
+        if isinstance(args.prompt_selection_percentage, bool) or not isinstance(
+            args.prompt_selection_percentage, int
+        ):
+            raise TypeError(
+                ERROR_BENCHMARK_RUN_COOKBOOK_PROMPT_SELECTION_PERCENTAGE_VALIDATION
+            )
+        elif (
+            args.prompt_selection_percentage < 1
+            or args.prompt_selection_percentage > 100
         ):
-            raise TypeError(ERROR_BENCHMARK_RUN_COOKBOOK_NUM_OF_PROMPTS_VALIDATION)
+            raise ValueError(
+                ERROR_BENCHMARK_RUN_COOKBOOK_PROMPT_SELECTION_PERCENTAGE_RANGE_VALIDATION
+            )
 
         if isinstance(args.random_seed, bool) or not isinstance(args.random_seed, int):
             raise TypeError(ERROR_BENCHMARK_RUN_COOKBOOK_RANDOM_SEED_VALIDATION)
@@ -297,7 +308,7 @@ def run_cookbook(args) -> None:
         async def run():
             await cb_runner.run_cookbooks(
                 cookbooks,
-                args.num_of_prompts,
+                args.prompt_selection_percentage,
                 args.random_seed,
                 args.system_prompt,
                 args.runner_proc_module,
@@ -436,9 +447,20 @@ def _display_cookbooks(cookbooks_list):
     table.add_column("Cookbook", justify="left", width=78)
     table.add_column("Contains", justify="left", width=20, overflow="fold")
     for idx, cookbook in enumerate(cookbooks_list, 1):
-        id, name, description, recipes, *other_args = cookbook.values()
+        (
+            id,
+            name,
+            tags,
+            categories,
+            description,
+            recipes,
+            *other_args,
+        ) = cookbook.values()
         idx = cookbook.get("idx", idx)
-        cookbook_info = f"[red]ID: {id}[/red]\n\n[blue]{name}[/blue]\n{description}"
+        cookbook_info = f"[red]ID: {id}[/red]\n\n[blue]{name}[/blue]\n\n{description}"
+        cookbook_info += (
+            f"\n\n[blue]Tags: {tags}[/blue]\n[blue]Categories: {categories}[/blue]\n"
+        )
         recipes_info = display_view_list_format("Recipes", recipes)
         table.add_section()
         table.add_row(str(idx), cookbook_info, recipes_info)
@@ -459,11 +481,11 @@ def _display_view_cookbook(cookbook_info):
     Returns:
         None
     """
-    id, name, description, recipes = cookbook_info.values()
+    id, name, tags, categories, description, recipes = cookbook_info.values()
     recipes_list = api_read_recipes(recipes)
     if recipes_list:
         table = Table(
-            title=f'Cookbook "{name}"',
+            title=f'Cookbook: "{name}"\n Tags: {tags}\n Categories: {categories}\n',
             show_lines=True,
             expand=True,
             header_style="bold",
@@ -471,6 +493,7 @@ def _display_view_cookbook(cookbook_info):
         table.add_column("No.", width=2)
         table.add_column("Recipe", justify="left", width=78)
         table.add_column("Contains", justify="left", width=20, overflow="fold")
+
         for recipe_id, recipe in enumerate(recipes_list, 1):
             (
                 id,
@@ -718,7 +741,11 @@ run_cookbook_args.add_argument("name", type=str, help="Name of cookbook runner")
 run_cookbook_args.add_argument("cookbooks", type=str, help="List of cookbooks to run")
 run_cookbook_args.add_argument("endpoints", type=str, help="List of endpoints to run")
 run_cookbook_args.add_argument(
-    "-n", "--num_of_prompts", type=int, default=0, help="Number of prompts to run"
+    "-n",
+    "--prompt_selection_percentage",
+    type=int,
+    default=100,
+    help="Percentage of prompts to run",
 )
 run_cookbook_args.add_argument(
     "-r", "--random_seed", type=int, default=0, help="Random seed number"

moonshot/integrations/cli/benchmark/recipe.py

@@ -40,7 +40,8 @@ from moonshot.integrations.cli.cli_errors import (
     ERROR_BENCHMARK_RUN_RECIPE_ENDPOINTS_VALIDATION_1,
     ERROR_BENCHMARK_RUN_RECIPE_NAME_VALIDATION,
     ERROR_BENCHMARK_RUN_RECIPE_NO_RESULT,
-    ERROR_BENCHMARK_RUN_RECIPE_NUM_OF_PROMPTS_VALIDATION,
+    ERROR_BENCHMARK_RUN_RECIPE_PROMPT_SELECTION_PERCENTAGE_RANGE_VALIDATION,
+    ERROR_BENCHMARK_RUN_RECIPE_PROMPT_SELECTION_PERCENTAGE_VALIDATION,
     ERROR_BENCHMARK_RUN_RECIPE_RANDOM_SEED_VALIDATION,
     ERROR_BENCHMARK_RUN_RECIPE_RECIPES_VALIDATION,
     ERROR_BENCHMARK_RUN_RECIPE_RECIPES_VALIDATION_1,
@@ -293,11 +294,12 @@ def run_recipe(args) -> None:
     The recipes are run against the specified endpoints, and the results are processed and displayed.
 
     Args:
-        args: A namespace object from argparse. It should have the following attributes:
+        args (argparse.Namespace): The arguments provided to the command line interface.
+        Expected keys are:
             name (str): The name of the recipe runner.
             recipes (str): A string representation of a list of recipes to run.
             endpoints (str): A string representation of a list of endpoints to run.
-            num_of_prompts (int): The number of prompts to run.
+            prompt_selection_percentage (int): The percentage of prompts to run.
             random_seed (int): The random seed number for reproducibility.
             system_prompt (str): The system prompt to use.
             runner_proc_module (str): The runner processing module to use.
@@ -329,10 +331,19 @@ def run_recipe(args) -> None:
         ):
             raise TypeError(ERROR_BENCHMARK_RUN_RECIPE_ENDPOINTS_VALIDATION)
 
-        if isinstance(args.num_of_prompts, bool) or not isinstance(
-            args.num_of_prompts, int
+        if isinstance(args.prompt_selection_percentage, bool) or not isinstance(
+            args.prompt_selection_percentage, int
+        ):
+            raise TypeError(
+                ERROR_BENCHMARK_RUN_RECIPE_PROMPT_SELECTION_PERCENTAGE_VALIDATION
+            )
+        elif (
+            args.prompt_selection_percentage < 1
+            or args.prompt_selection_percentage > 100
         ):
-            raise TypeError(ERROR_BENCHMARK_RUN_RECIPE_NUM_OF_PROMPTS_VALIDATION)
+            raise ValueError(
+                ERROR_BENCHMARK_RUN_RECIPE_PROMPT_SELECTION_PERCENTAGE_RANGE_VALIDATION
+            )
 
         if isinstance(args.random_seed, bool) or not isinstance(args.random_seed, int):
             raise TypeError(ERROR_BENCHMARK_RUN_RECIPE_RANDOM_SEED_VALIDATION)
@@ -377,7 +388,7 @@ def run_recipe(args) -> None:
         async def run():
             await rec_runner.run_recipes(
                 recipes,
-                args.num_of_prompts,
+                args.prompt_selection_percentage,
                 args.random_seed,
                 args.system_prompt,
                 args.runner_proc_module,
@@ -809,7 +820,11 @@ run_recipe_args.add_argument("name", type=str, help="Name of recipe runner")
 run_recipe_args.add_argument("recipes", type=str, help="List of recipes to run")
 run_recipe_args.add_argument("endpoints", type=str, help="List of endpoints to run")
 run_recipe_args.add_argument(
-    "-n", "--num_of_prompts", type=int, default=0, help="Number of prompts to run"
+    "-n",
+    "--prompt_selection_percentage",
+    type=int,
+    default=100,
+    help="Percentage of prompts to run",
 )
 run_recipe_args.add_argument(
     "-r", "--random_seed", type=int, default=0, help="Random seed number"

moonshot/integrations/cli/benchmark/result.py

@@ -190,7 +190,7 @@ def _display_results(results_list):
         recipes = metadata["recipes"]
         cookbooks = metadata["cookbooks"]
         endpoints = metadata["endpoints"]
-        num_of_prompts = metadata["num_of_prompts"]
+        prompt_selection_percentage = metadata["prompt_selection_percentage"]
         random_seed = metadata["random_seed"]
         system_prompt = metadata["system_prompt"]
         idx = result.get("idx", idx)
@@ -200,7 +200,9 @@ def _display_results(results_list):
         recipes_info = display_view_list_format("Recipes", recipes)
         cookbooks_info = display_view_list_format("Cookbooks", cookbooks)
         endpoints_info = display_view_list_format("Endpoints", endpoints)
-        prompts_info = display_view_str_format("Number of Prompts", num_of_prompts)
+        prompts_info = display_view_str_format(
+            "Prompt Selection Percentage", prompt_selection_percentage
+        )
         seed_info = display_view_str_format("Seed", random_seed)
         system_prompt_info = display_view_str_format("System Prompt", system_prompt)
 

moonshot/integrations/cli/cli_errors.py

@@ -52,8 +52,11 @@ ERROR_BENCHMARK_RUN_COOKBOOK_ENDPOINTS_VALIDATION = (
 ERROR_BENCHMARK_RUN_COOKBOOK_ENDPOINTS_VALIDATION_1 = (
     "The 'endpoints' argument must evaluate to a list of strings."
 )
-ERROR_BENCHMARK_RUN_COOKBOOK_NUM_OF_PROMPTS_VALIDATION = (
-    "The 'num_of_prompts' argument must be an integer."
+ERROR_BENCHMARK_RUN_COOKBOOK_PROMPT_SELECTION_PERCENTAGE_VALIDATION = (
+    "The 'prompt_selection_percentage' argument must be an integer."
+)
+ERROR_BENCHMARK_RUN_COOKBOOK_PROMPT_SELECTION_PERCENTAGE_RANGE_VALIDATION = (
+    "The 'prompt_selection_percentage' argument must be between 1 - 100."
 )
 ERROR_BENCHMARK_RUN_COOKBOOK_RANDOM_SEED_VALIDATION = (
     "The 'random_seed' argument must be an integer."
@@ -278,8 +281,11 @@ ERROR_BENCHMARK_RUN_RECIPE_ENDPOINTS_VALIDATION = (
 ERROR_BENCHMARK_RUN_RECIPE_ENDPOINTS_VALIDATION_1 = (
     "The 'endpoints' argument must evaluate to a list of strings."
 )
-ERROR_BENCHMARK_RUN_RECIPE_NUM_OF_PROMPTS_VALIDATION = (
-    "The 'num_of_prompts' argument must be an integer."
+ERROR_BENCHMARK_RUN_RECIPE_PROMPT_SELECTION_PERCENTAGE_VALIDATION = (
+    "The 'prompt_selection_percentage' argument must be an integer."
+)
+ERROR_BENCHMARK_RUN_RECIPE_PROMPT_SELECTION_PERCENTAGE_RANGE_VALIDATION = (
+    "The 'prompt_selection_percentage' argument must be between 1 - 100."
 )
 ERROR_BENCHMARK_RUN_RECIPE_RANDOM_SEED_VALIDATION = (
     "The 'random_seed' argument must be an integer."

moonshot/integrations/web_api/app.py

@@ -71,7 +71,7 @@ def create_app(cfg: providers.Configuration) -> CustomFastAPI:
     }
 
     app: CustomFastAPI = CustomFastAPI(
-        title="Project Moonshot", version="0.5.1", **app_kwargs
+        title="Project Moonshot", version="0.6.1", **app_kwargs
     )
 
     if cfg.cors.enabled():

moonshot/integrations/web_api/schemas/benchmark_runner_dto.py

@@ -1,4 +1,4 @@
-from pydantic import BaseModel, ConfigDict
+from pydantic import BaseModel, ConfigDict, Field
 
 
 class BenchmarkRunnerDTO(BaseModel):
@@ -7,7 +7,7 @@ class BenchmarkRunnerDTO(BaseModel):
     description: str
     endpoints: list[str]
     inputs: list[str]
-    num_of_prompts: int
+    prompt_selection_percentage: int = Field(..., ge=1, le=100)
     random_seed: int
     system_prompt: str
     runner_processing_module: str

moonshot/integrations/web_api/schemas/cookbook_create_dto.py

@@ -9,6 +9,8 @@ class CookbookCreateDTO(CookbookPydanticModel):
     id: Optional[str] = None
     name: str = Field(..., min_length=1)
     description: Optional[str] = Field(default="", min_length=1)
+    tags: Optional[list[str]] = []
+    categories: Optional[list[str]] = []
     recipes: list[str] = Field(..., min_length=1)
 
 
@@ -16,4 +18,6 @@ class CookbookUpdateDTO(CookbookPydanticModel):
     id: Optional[str] = None
     name: Optional[str] = Field(default=None, min_length=1)
     description: Optional[str] = Field(default=None, min_length=1)
+    tags: Optional[list[str]] = None
+    categories: Optional[list[str]] = None
     recipes: Optional[list[str]] = Field(default=None, min_length=1)

moonshot/integrations/web_api/schemas/dataset_create_dto.py

@@ -1,7 +1,6 @@
-from typing import Optional
+from typing import Any, Optional
 
 from pydantic import Field
-from pyparsing import Iterator
 
 from moonshot.src.datasets.dataset_arguments import (
     DatasetArguments as DatasetPydanticModel,
@@ -10,7 +9,7 @@ from moonshot.src.datasets.dataset_arguments import (
 
 class CSV_Dataset_DTO(DatasetPydanticModel):
     id: Optional[str] = None  # Not a required from user
-    examples: Optional[Iterator[dict]] = None  # Not a required from user
+    examples: Optional[Any] = None  # Not a required from user
     name: str = Field(..., min_length=1)
     description: str = Field(default="", min_length=1)
     license: Optional[str] = ""
@@ -20,7 +19,7 @@ class CSV_Dataset_DTO(DatasetPydanticModel):
 
 class HF_Dataset_DTO(DatasetPydanticModel):
     id: Optional[str] = None  # Not a required from user
-    examples: Optional[Iterator[dict]] = None  # Not a required from user
+    examples: Optional[Any] = None  # Not a required from user
     name: str = Field(..., min_length=1)
     description: str = Field(default="", min_length=1)
     license: Optional[str] = ""

moonshot/integrations/web_api/services/benchmark_test_manager.py

@@ -60,14 +60,14 @@ class BenchmarkTestManager(BaseService):
             if benchmark_type == BenchmarkCollectionType.COOKBOOK:
                 async_run = moonshot_runner.run_cookbooks(
                     cookbooks=benchmark_input_data.inputs,
-                    num_of_prompts=benchmark_input_data.num_of_prompts,
+                    prompt_selection_percentage=benchmark_input_data.prompt_selection_percentage,
                     random_seed=benchmark_input_data.random_seed,
                     system_prompt=benchmark_input_data.system_prompt,
                 )
             else:
                 async_run = moonshot_runner.run_recipes(
                     recipes=benchmark_input_data.inputs,
-                    num_of_prompts=benchmark_input_data.num_of_prompts,
+                    prompt_selection_percentage=benchmark_input_data.prompt_selection_percentage,
                     random_seed=benchmark_input_data.random_seed,
                     system_prompt=benchmark_input_data.system_prompt,
                 )

moonshot/integrations/web_api/services/cookbook_service.py

@@ -71,7 +71,7 @@ class CookbookService(BaseService):
                             cookbook.total_dataset_in_cookbook,
                         ) = get_total_prompt_and_dataset_in_cookbook(cookbook)
 
-            if tags and cookbooks_recipe_has_tags(tags, cookbook):
+            if tags and cookbook_has_tags(tags, cookbook):
                 if cookbook not in cookbooks_list:
                     cookbooks_list.append(cookbook)
                     if count:
@@ -80,7 +80,7 @@ class CookbookService(BaseService):
                             cookbook.total_dataset_in_cookbook,
                         ) = get_total_prompt_and_dataset_in_cookbook(cookbook)
 
-            if categories and cookbooks_recipe_has_categories(categories, cookbook):
+            if categories and cookbook_has_categories(categories, cookbook):
                 if cookbook not in cookbooks_list:
                     cookbooks_list.append(cookbook)
                     if count:
@@ -89,10 +89,16 @@ class CookbookService(BaseService):
                             cookbook.total_dataset_in_cookbook,
                         ) = get_total_prompt_and_dataset_in_cookbook(cookbook)
 
-            if categories_excluded and cookbooks_recipe_has_categories(
-                categories_excluded, cookbook
-            ):
-                cookbooks_list.remove(cookbook)
+            if categories_excluded:
+                excluded_categories_set = set(
+                    category.lower() for category in categories_excluded.split(",")
+                )
+                cookbook_categories_set = set(
+                    category.lower() for category in cookbook.categories
+                )
+                # Exclude only if all categories in the cookbook are in the excluded list
+                if cookbook_categories_set.issubset(excluded_categories_set):
+                    cookbooks_list.remove(cookbook)
 
         for cookbook in cookbooks_list:
             cookbook.required_config = cookbook_metrics_dependency(cookbook)
@@ -160,50 +166,40 @@ def get_total_prompt_and_dataset_in_cookbook(cookbook: Cookbook) -> tuple[int, i
 
 
 @staticmethod
-def cookbooks_recipe_has_tags(tags: str, cookbook: Cookbook) -> bool:
+def cookbook_has_tags(tags: str, cookbook: Cookbook) -> bool:
     """
-    Check if any recipe in a cookbook has the specified tags.
+    Check if a cookbook has the specified tags.
 
     Args:
-        tags (str): The tags to check for in the cookbook's recipes.
-        cookbook (Cookbook): The cookbook object containing the recipe IDs.
+        tags (str): The tags to check for in the cookbook.
+        cookbook (Cookbook): The cookbook object.
 
     Returns:
-        bool: True if any recipe in the cookbook has the specified tags, False otherwise.
+        bool: True if the cookbook has the specified tags, False otherwise.
     """
-    recipe_ids = cookbook.recipes
-    recipes = moonshot_api.api_read_recipes(recipe_ids)
-    for recipe in recipes:
-        recipe = Recipe(**recipe)
-        if tags in recipe.tags:
-            return True
-    return False
+    tags_list = [tag.lower() for tag in tags.split(",")]
+    return any(tag in [ctag.lower() for ctag in cookbook.tags] for tag in tags_list)
 
 
 @staticmethod
-def cookbooks_recipe_has_categories(categories: str, cookbook: Cookbook) -> bool:
+def cookbook_has_categories(categories: str, cookbook: Cookbook) -> bool:
     """
-    Check if any recipe in a cookbook has the specified categories.
+    Check if a cookbook has the specified categories.
 
     Args:
-        categories (str): The categories to check for in the cookbook's recipes.
-        cookbook (Cookbook): The cookbook object containing the recipe IDs.
-        exclude_categories (str): The categories to exclude
+        categories (str): The categories to check for in the cookbook.
+        cookbook (Cookbook): The cookbook object.
 
     Returns:
-        bool: True if any recipe in the cookbook has the specified categories, False otherwise.
+        bool: True if the cookbook has the specified categories, False otherwise.
     """
-    recipe_ids = cookbook.recipes
     categories_list = [category.lower() for category in categories.split(",")]
-    recipes = moonshot_api.api_read_recipes(recipe_ids)
-    for recipe in recipes:
-        recipe = Recipe(**recipe)
-        if any(
-            category in [rcat.lower() for rcat in recipe.categories]
-            for category in categories_list
-        ):
-            return True
-    return False
+    return any(
+        category in [ccat.lower() for ccat in cookbook.categories]
+        for category in categories_list
+    )
+
+
 
 
 @staticmethod

moonshot/integrations/web_api/types/types.py

@@ -85,7 +85,7 @@ class ResultMetadata(TypedDict):
     recipes: List[str]
     cookbooks: List[str]
     endpoints: List[str]
-    num_of_prompts: int
+    prompt_selection_percentage: int
     status: str
 
 

moonshot/src/api/api_cookbook.py

@@ -2,6 +2,7 @@ from pydantic import conlist, validate_call
 
 from moonshot.src.cookbooks.cookbook import Cookbook
 from moonshot.src.cookbooks.cookbook_arguments import CookbookArguments
+from moonshot.src.recipes.recipe import Recipe
 
 
 # ------------------------------------------------------------------------------
@@ -20,6 +21,8 @@ def api_create_cookbook(name: str, description: str, recipes: list[str]) -> str:
     Args:
         name (str): The name of the new cookbook.
         description (str): A brief description of the new cookbook.
+        tags (list[str]): A list of tags associated with the cookbook.
+        categories (list[str]): A list of categories the cookbook belongs to.
         recipes (list[str]): A list of recipes to be included in the new cookbook.
 
     Returns:
@@ -29,10 +32,13 @@ def api_create_cookbook(name: str, description: str, recipes: list[str]) -> str:
     # We do not need to provide the id.
     # This is because during creation:
     # 1. the id is slugify from the name and stored as id.
+    # We do not need to provide tags and categories as they will be generated based on the recipes selected.
     cb_args = CookbookArguments(
         id="",
         name=name,
         description=description,
+        tags=[],
+        categories=[],
         recipes=recipes,
     )
     return Cookbook.create(cb_args)
@@ -103,6 +109,20 @@ def api_update_cookbook(cb_id: str, **kwargs) -> bool:
         if hasattr(existing_cookbook, key):
             setattr(existing_cookbook, key, value)
 
+    # Update the cookbook's categories and tags if any of the recipe(s) are changed
+    if "recipes" in kwargs:
+        consolidated_tags = set()
+        consolidated_categories = set()
+        for key, value in kwargs.items():
+            if key == "recipes":
+                for recipe_id in value:
+                    recipe = Recipe.read(recipe_id)
+                    consolidated_tags.update(recipe.tags)
+                    consolidated_categories.update(recipe.categories)
+        # Consolidate and set the tags and categories
+        existing_cookbook.tags = list(consolidated_tags)
+        existing_cookbook.categories = list(consolidated_categories)
+
     # Perform pydantic check on the updated existing cookbook
     CookbookArguments.model_validate(existing_cookbook.to_dict())
 

moonshot/src/cookbooks/cookbook.py

@@ -7,6 +7,7 @@ from slugify import slugify
 
 from moonshot.src.configs.env_variables import EnvVariables
 from moonshot.src.cookbooks.cookbook_arguments import CookbookArguments
+from moonshot.src.recipes.recipe import Recipe
 from moonshot.src.storage.storage import Storage
 from moonshot.src.utils.log import configure_logger
 
@@ -19,6 +20,8 @@ class Cookbook:
         self.id = cb_args.id
         self.name = cb_args.name
         self.description = cb_args.description
+        self.tags = cb_args.tags
+        self.categories = cb_args.categories
         self.recipes = cb_args.recipes
 
     @classmethod
@@ -64,6 +67,8 @@ class Cookbook:
             cb_info = {
                 "name": cb_args.name,
                 "description": cb_args.description,
+                "tags": Cookbook.get_tags_in_recipes(cb_args.recipes),
+                "categories": Cookbook.get_categories_in_recipes(cb_args.recipes),
                 "recipes": cb_args.recipes,
             }
 
@@ -243,3 +248,19 @@ class Cookbook:
         except Exception as e:
             logger.error(f"Failed to get available cookbooks: {str(e)}")
             raise e
+
+    @staticmethod
+    def get_categories_in_recipes(recipes: list[str]) -> list[str]:
+        return list(
+            {
+                category
+                for recipe_id in recipes
+                for category in Recipe.read(recipe_id).categories
+            }
+        )
+
+    @staticmethod
+    def get_tags_in_recipes(recipes: list[str]) -> list[str]:
+        return list(
+            {tag for recipe_id in recipes for tag in Recipe.read(recipe_id).tags}
+        )

moonshot/src/cookbooks/cookbook_arguments.py

@@ -8,6 +8,10 @@ class CookbookArguments(BaseModel):
 
     description: str  # description (str): A brief description of the Cookbook.
 
+    tags: list[str]  #  tags (list): The list of tags in the Cookbook.
+
+    categories: list[str]  # categories (list): The list of categories in the Cookbook.
+
     recipes: list[str] = Field(
         min_length=1
     )  # recipes (list): A list of recipes included in the Cookbook.
@@ -29,6 +33,8 @@ class CookbookArguments(BaseModel):
         return {
             "id": self.id,
             "name": self.name,
+            "tags": self.tags,
+            "categories": self.categories,
             "description": self.description,
             "recipes": self.recipes,
         }

moonshot/src/runners/runner.py

@@ -315,7 +315,7 @@ class Runner:
     async def run_recipes(
         self,
         recipes: list[str],
-        num_of_prompts: int = 0,
+        prompt_selection_percentage: int = 100,
         random_seed: int = 0,
         system_prompt: str = "",
         runner_processing_module: str = "benchmarking",
@@ -325,26 +325,21 @@ class Runner:
         Initiates an asynchronous benchmark run using a set of recipes.
 
         This method sets up and starts a benchmark run tailored for recipes. It instantiates a benchmark run object,
-        applies the configuration based on the provided recipes, number of prompts, random seed, system prompt, and
+        applies the configuration based on the provided recipes, percentage of prompts, random seed, system prompt, and
         the specified runner and result processing modules, and then commences the run asynchronously.
 
         Args:
             recipes (list[str]): The recipes to be included in the benchmark run.
-
-            num_of_prompts (int, optional): The count of prompts to utilize during the benchmark.
-            Defaults to 0.
-
+            prompt_selection_percentage (int, optional): The percentage of prompts to utilize during the benchmark.
+                Defaults to 100.
             random_seed (int, optional): The seed for random number generation to ensure reproducibility.
-            Defaults to 0.
-
+                Defaults to 0.
             system_prompt (str, optional): The system prompt to be used during the benchmark.
-            Defaults to an empty string.
-
+                Defaults to an empty string.
             runner_processing_module (str, optional): The module responsible for processing the runner.
-            Defaults to "benchmarking".
-
+                Defaults to "benchmarking".
             result_processing_module (str, optional): The module responsible for processing the results.
-            Defaults to "benchmarking-result".
+                Defaults to "benchmarking-result".
 
         Raises:
             Exception: If any error occurs during the setup or execution of the benchmark run.
@@ -357,7 +352,7 @@ class Runner:
                 RunnerType.BENCHMARK,
                 {
                     "recipes": recipes,
-                    "num_of_prompts": num_of_prompts,
+                    "prompt_selection_percentage": prompt_selection_percentage,
                     "random_seed": random_seed,
                     "system_prompt": system_prompt,
                     "runner_processing_module": runner_processing_module,
@@ -382,7 +377,7 @@ class Runner:
     async def run_cookbooks(
         self,
         cookbooks: list[str],
-        num_of_prompts: int = 0,
+        prompt_selection_percentage: int = 100,
         random_seed: int = 0,
         system_prompt: str = "",
         runner_processing_module: str = "benchmarking",
@@ -393,26 +388,21 @@ class Runner:
 
         This method is responsible for initiating a benchmark cookbook run with the specified cookbooks and parameters.
         It creates a new benchmark cookbook run instance, configures it with the provided cookbook names,
-        number of prompts, random seed, system prompt, runner processing module, and result processing module,
+        percentage of prompts, random seed, system prompt, runner processing module, and result processing module,
         and then starts the run asynchronously.
 
         Args:
             cookbooks (list[str]): A list of cookbook names to be run in the benchmark.
-
-            num_of_prompts (int, optional): The number of prompts to be used in the benchmark run.
-            Defaults to 0.
-
+            prompt_selection_percentage (int, optional): The percentage of prompts to be used in the benchmark run.
+                Defaults to 100.
             random_seed (int, optional): The seed for random number generation to ensure reproducibility.
-            Defaults to 0.
-
+                Defaults to 0.
             system_prompt (str, optional): A system prompt to be used in the benchmark run.
-            Defaults to an empty string.
-
+                Defaults to an empty string.
             runner_processing_module (str, optional): The module responsible for processing the runner.
-            Defaults to "benchmarking".
-
+                Defaults to "benchmarking".
             result_processing_module (str, optional): The module responsible for processing the results.
-            Defaults to "benchmarking-result".
+                Defaults to "benchmarking-result".
 
         Raises:
             Exception: If any error occurs during the setup or execution of the benchmark run.
@@ -425,7 +415,7 @@ class Runner:
                 RunnerType.BENCHMARK,
                 {
                     "cookbooks": cookbooks,
-                    "num_of_prompts": num_of_prompts,
+                    "prompt_selection_percentage": prompt_selection_percentage,
                     "random_seed": random_seed,
                     "system_prompt": system_prompt,
                     "runner_processing_module": runner_processing_module,