aiverify-moonshot 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

moonshot/integrations/cli/benchmark/recipe.py

@@ -18,6 +18,7 @@ from moonshot.api import (
     api_update_recipe,
 )
 from moonshot.integrations.cli.common.display_helper import display_view_list_format
+from moonshot.integrations.cli.utils.process_data import filter_data
 
 console = Console()
 
@@ -36,8 +37,7 @@ def add_recipe(args) -> None:
 
     Args:
         args (argparse.Namespace): The arguments provided to the command line interface.
-        Expected keys are name, description, tags, categories, dataset, prompt_templates, metrics, attack_modules,
-        and grading_scale.
+        Expected keys are name, description, tags, categories, dataset, prompt_templates, metrics and grading_scale.
 
     Returns:
         None
@@ -53,9 +53,6 @@ def add_recipe(args) -> None:
             literal_eval(args.prompt_templates) if args.prompt_templates else []
         )
         metrics = literal_eval(args.metrics)
-        attack_modules = (
-            literal_eval(args.attack_modules) if args.attack_modules else []
-        )
         grading_scale = literal_eval(args.grading_scale) if args.grading_scale else {}
 
         new_recipe_id = api_create_recipe(
@@ -66,7 +63,6 @@ def add_recipe(args) -> None:
             datasets,
             prompt_templates,
             metrics,
-            attack_modules,
             grading_scale,
         )
         print(f"[add_recipe]: Recipe ({new_recipe_id}) created.")
@@ -74,20 +70,37 @@ def add_recipe(args) -> None:
         print(f"[add_recipe]: {str(e)}")
 
 
-def list_recipes() -> None:
+def list_recipes(args) -> list | None:
     """
     List all available recipes.
 
     This function retrieves all available recipes by calling the api_get_all_recipe function from the
     moonshot.api module.
-    It then displays the retrieved recipes using the display_recipes function.
+    It then displays the retrieved recipes using the _display_recipes function.
+
+    Args:
+        args: A namespace object from argparse. It should have an optional attribute:
+        find (str): Optional field to find recipe(s) with a keyword.
+        pagination (str): Optional field to paginate recipes.
 
     Returns:
-        None
+        list | None: A list of Recipe or None if there is no result.
     """
+
     try:
         recipes_list = api_get_all_recipe()
-        display_recipes(recipes_list)
+        keyword = args.find.lower() if args.find else ""
+        pagination = literal_eval(args.pagination) if args.pagination else ()
+
+        if recipes_list:
+            filtered_recipes_list = filter_data(recipes_list, keyword, pagination)
+            if filtered_recipes_list:
+                _display_recipes(filtered_recipes_list)
+                return filtered_recipes_list
+
+        console.print("[red]There are no recipes found.[/red]")
+        return None
+
     except Exception as e:
         print(f"[list_recipes]: {str(e)}")
 
@@ -109,7 +122,7 @@ def view_recipe(args) -> None:
     """
     try:
         recipe_info = api_read_recipe(args.recipe)
-        display_recipes([recipe_info])
+        _display_recipes([recipe_info])
     except Exception as e:
         print(f"[view_recipe]: {str(e)}")
 
@@ -295,7 +308,7 @@ def display_view_statistics_format(title: str, stats: dict) -> str:
         return f"[blue]{title}[/blue]: nil"
 
 
-def display_recipes(recipes_list: list) -> None:
+def _display_recipes(recipes_list: list) -> None:
     """
     Display the list of recipes in a tabular format.
 
@@ -307,54 +320,48 @@ def display_recipes(recipes_list: list) -> None:
     Args:
         recipes_list (list): A list of dictionaries, where each dictionary contains the details of a recipe.
     """
-    if recipes_list:
-        table = Table(
-            title="List of Recipes", show_lines=True, expand=True, header_style="bold"
+    table = Table(
+        title="List of Recipes", show_lines=True, expand=True, header_style="bold"
+    )
+    table.add_column("No.", width=2)
+    table.add_column("Recipe", justify="left", width=78)
+    table.add_column("Contains", justify="left", width=20, overflow="fold")
+    for idx, recipe in enumerate(recipes_list, 1):
+        (
+            id,
+            name,
+            description,
+            tags,
+            categories,
+            datasets,
+            prompt_templates,
+            metrics,
+            grading_scale,
+            stats,
+            *other_args,
+        ) = recipe.values()
+        idx = recipe.get("idx", idx)
+        tags_info = display_view_list_format("Tags", tags)
+        categories_info = display_view_list_format("Categories", categories)
+        datasets_info = display_view_list_format("Datasets", datasets)
+        prompt_templates_info = display_view_list_format(
+            "Prompt Templates", prompt_templates
         )
-        table.add_column("No.", width=2)
-        table.add_column("Recipe", justify="left", width=78)
-        table.add_column("Contains", justify="left", width=20, overflow="fold")
-        for recipe_id, recipe in enumerate(recipes_list, 1):
-            (
-                id,
-                name,
-                description,
-                tags,
-                categories,
-                datasets,
-                prompt_templates,
-                metrics,
-                attack_strategies,
-                grading_scale,
-                stats,
-            ) = recipe.values()
-
-            tags_info = display_view_list_format("Tags", tags)
-            categories_info = display_view_list_format("Categories", categories)
-            datasets_info = display_view_list_format("Datasets", datasets)
-            prompt_templates_info = display_view_list_format(
-                "Prompt Templates", prompt_templates
-            )
-            metrics_info = display_view_list_format("Metrics", metrics)
-            attack_strategies_info = display_view_list_format(
-                "Attack Strategies", attack_strategies
-            )
-            grading_scale_info = display_view_grading_scale_format(
-                "Grading Scale", grading_scale
-            )
-            stats_info = display_view_statistics_format("Statistics", stats)
+        metrics_info = display_view_list_format("Metrics", metrics)
+        grading_scale_info = display_view_grading_scale_format(
+            "Grading Scale", grading_scale
+        )
+        stats_info = display_view_statistics_format("Statistics", stats)
 
-            recipe_info = (
-                f"[red]id: {id}[/red]\n\n[blue]{name}[/blue]\n{description}\n\n"
-                f"{tags_info}\n\n{categories_info}\n\n{grading_scale_info}\n\n{stats_info}"
-            )
-            contains_info = f"{datasets_info}\n\n{prompt_templates_info}\n\n{metrics_info}\n\n{attack_strategies_info}"
+        recipe_info = (
+            f"[red]id: {id}[/red]\n\n[blue]{name}[/blue]\n{description}\n\n"
+            f"{tags_info}\n\n{categories_info}\n\n{grading_scale_info}\n\n{stats_info}"
+        )
+        contains_info = f"{datasets_info}\n\n{prompt_templates_info}\n\n{metrics_info}"
 
-            table.add_section()
-            table.add_row(str(recipe_id), recipe_info, contains_info)
-        console.print(table)
-    else:
-        console.print("[red]There are no recipes found.[/red]")
+        table.add_section()
+        table.add_row(str(idx), recipe_info, contains_info)
+    console.print(table)
 
 
 def show_recipe_results(recipes, endpoints, recipe_results, duration):
@@ -382,9 +389,9 @@ def show_recipe_results(recipes, endpoints, recipe_results, duration):
         console.print("[red]There are no results.[/red]")
 
     # Print run stats
-    console.print(
-        f"{'='*50}\n[blue]Time taken to run: {duration}s[/blue]\n*Overall rating will be the lowest grade that the recipes have in each cookbook\n{'='*50}"
-    )
+    run_stats = f"""{'='*50}\n[blue]Time taken to run: {duration}s[/blue]\n*Overall rating will be the lowest grade
+    that the recipes have in each cookbook\n{'='*50}"""
+    console.print(run_stats)
 
 
 def generate_recipe_table(recipes: list, endpoints: list, results: dict) -> None:
@@ -483,7 +490,6 @@ add_recipe_args = cmd2.Cmd2ArgumentParser(
     "\"['bertscore','bleuscore']\" "
     "-p \"['analogical-similarity','mmlu']\" "
     "-t \"['tag1','tag2']\" "
-    "-a \"['charswap_attack']\" "
     "-g \"{'A':[80,100],'B':[60,79],'C':[40,59],'D':[20,39],'E':[0,19]}\" ",
 )
 add_recipe_args.add_argument("name", type=str, help="Name of the new recipe")
@@ -511,13 +517,6 @@ add_recipe_args.add_argument(
 add_recipe_args.add_argument(
     "metrics", type=str, help="List of metrics to be included in the new recipe"
 )
-add_recipe_args.add_argument(
-    "-a",
-    "--attack_modules",
-    type=str,
-    help="List of attack modules to be included in the new recipe",
-    nargs="?",
-)
 add_recipe_args.add_argument(
     "-g",
     "--grading_scale",
@@ -537,7 +536,6 @@ update_recipe_args = cmd2.Cmd2ArgumentParser(
     "  datasets: A list of datasets used in the recipe. \n"
     "  prompt_templates: A list of prompt templates for the recipe. \n"
     "  metrics: A list of metrics to evaluate the recipe. \n"
-    "  attack_modules: A list of attack modules used in the recipe.\n"
     "  grading_scale: A list of grading scale used in the recipe. \n\n"
     "Example command:\n"
     "  update_recipe my-new-recipe \"[('name', 'My Updated Recipe'), ('tags', ['fairness', 'bbq'])]\" ",
@@ -596,3 +594,25 @@ run_recipe_args.add_argument(
     default="benchmarking-result",
     help="Result processing module to use",
 )
+
+# List recipe arguments
+list_recipes_args = cmd2.Cmd2ArgumentParser(
+    description="List all recipes.",
+    epilog='Example:\n list_recipes -f "mmlu"',
+)
+
+list_recipes_args.add_argument(
+    "-f",
+    "--find",
+    type=str,
+    help="Optional field to find recipe(s) with keyword",
+    nargs="?",
+)
+
+list_recipes_args.add_argument(
+    "-p",
+    "--pagination",
+    type=str,
+    help="Optional tuple to paginate recipes(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
+    nargs="?",
+)

moonshot/integrations/cli/benchmark/result.py

@@ -1,3 +1,5 @@
+from ast import literal_eval
+
 import cmd2
 from rich.console import Console
 from rich.table import Table
@@ -9,6 +11,7 @@ from moonshot.integrations.cli.common.display_helper import (
     display_view_list_format,
     display_view_str_format,
 )
+from moonshot.integrations.cli.utils.process_data import filter_data
 
 console = Console()
 
@@ -16,7 +19,7 @@ console = Console()
 # ------------------------------------------------------------------------------
 # CLI Functions
 # ------------------------------------------------------------------------------
-def list_results() -> None:
+def list_results(args) -> list | None:
     """
     List all available results.
 
@@ -24,12 +27,29 @@ def list_results() -> None:
     moonshot.api module. It then creates a table with the result id and name. If there are no results, it prints a
     message indicating that no results were found.
 
+    Args:
+        args: A namespace object from argparse. It should have an optional attribute:
+        find (str): Optional field to find result(s) with a keyword.
+        pagination (str): Optional field to paginate results.
+
     Returns:
-        None
+        list | None: A list of Result or None if there is no result.
     """
+
     try:
         results_list = api_get_all_result()
-        display_results(results_list)
+        keyword = args.find.lower() if args.find else ""
+        pagination = literal_eval(args.pagination) if args.pagination else ()
+
+        if results_list:
+            filtered_results_list = filter_data(results_list, keyword, pagination)
+            if filtered_results_list:
+                _display_results(filtered_results_list)
+                return filtered_results_list
+
+        console.print("[red]There are no results found.[/red]")
+        return None
+
     except Exception as e:
         print(f"[list_results]: {str(e)}")
 
@@ -95,7 +115,7 @@ def delete_result(args) -> None:
 # ------------------------------------------------------------------------------
 # Helper functions: Display on cli
 # ------------------------------------------------------------------------------
-def display_results(results_list):
+def _display_results(results_list):
     """
     Display a list of results.
 
@@ -108,50 +128,46 @@ def display_results(results_list):
     Returns:
         None
     """
-    if results_list:
-        table = Table(
-            title="List of Results", show_lines=True, expand=True, header_style="bold"
+    table = Table(
+        title="List of Results", show_lines=True, expand=True, header_style="bold"
+    )
+    table.add_column("No.", width=2)
+    table.add_column("Result", justify="left", width=78)
+    table.add_column("Contains", justify="left", width=20, overflow="fold")
+    for idx, result in enumerate(results_list, 1):
+        metadata, results, *other_args = result.values()
+
+        id = metadata["id"]
+        start_time = metadata["start_time"]
+        end_time = metadata["end_time"]
+        duration = metadata["duration"]
+        status = metadata["status"]
+        recipes = metadata["recipes"]
+        cookbooks = metadata["cookbooks"]
+        endpoints = metadata["endpoints"]
+        num_of_prompts = metadata["num_of_prompts"]
+        random_seed = metadata["random_seed"]
+        system_prompt = metadata["system_prompt"]
+        idx = result.get("idx", idx)
+
+        duration_info = f"[blue]Period:[/blue] {start_time} - {end_time} ({duration}s)"
+        status_info = display_view_str_format("Status", status)
+        recipes_info = display_view_list_format("Recipes", recipes)
+        cookbooks_info = display_view_list_format("Cookbooks", cookbooks)
+        endpoints_info = display_view_list_format("Endpoints", endpoints)
+        prompts_info = display_view_str_format("Number of Prompts", num_of_prompts)
+        seed_info = display_view_str_format("Seed", random_seed)
+        system_prompt_info = display_view_str_format("System Prompt", system_prompt)
+
+        result_info = f"[red]id: {id}[/red]\n\n{duration_info}\n\n{status_info}"
+        contains_info = (
+            f"{recipes_info}\n\n{cookbooks_info}\n\n{endpoints_info}\n\n{prompts_info}"
+            f"\n\n{seed_info}\n\n{system_prompt_info}"
         )
-        table.add_column("No.", width=2)
-        table.add_column("Result", justify="left", width=78)
-        table.add_column("Contains", justify="left", width=20, overflow="fold")
-        for result_id, result in enumerate(results_list, 1):
-            metadata, results = result.values()
-
-            id = metadata["id"]
-            start_time = metadata["start_time"]
-            end_time = metadata["end_time"]
-            duration = metadata["duration"]
-            status = metadata["status"]
-            recipes = metadata["recipes"]
-            cookbooks = metadata["cookbooks"]
-            endpoints = metadata["endpoints"]
-            num_of_prompts = metadata["num_of_prompts"]
-            random_seed = metadata["random_seed"]
-            system_prompt = metadata["system_prompt"]
-
-            duration_info = (
-                f"[blue]Period:[/blue] {start_time} - {end_time} ({duration}s)"
-            )
-            status_info = display_view_str_format("Status", status)
-            recipes_info = display_view_list_format("Recipes", recipes)
-            cookbooks_info = display_view_list_format("Cookbooks", cookbooks)
-            endpoints_info = display_view_list_format("Endpoints", endpoints)
-            prompts_info = display_view_str_format("Number of Prompts", num_of_prompts)
-            seed_info = display_view_str_format("Seed", random_seed)
-            system_prompt_info = display_view_str_format("System Prompt", system_prompt)
-
-            result_info = f"[red]id: {id}[/red]\n\n{duration_info}\n\n{status_info}"
-            contains_info = (
-                f"{recipes_info}\n\n{cookbooks_info}\n\n{endpoints_info}\n\n{prompts_info}"
-                f"\n\n{seed_info}\n\n{system_prompt_info}"
-            )
-
-            table.add_section()
-            table.add_row(str(result_id), result_info, contains_info)
-        console.print(table)
-    else:
-        console.print("[red]There are no results found.[/red]")
+
+        table.add_section()
+        table.add_row(str(idx), result_info, contains_info)
+    console.print(table)
 
 
 def display_view_recipe_result(result_info):
@@ -214,3 +230,25 @@ delete_result_args = cmd2.Cmd2ArgumentParser(
     epilog="Example:\n delete_result my-new-cookbook-runner",
 )
 delete_result_args.add_argument("result", type=str, help="Name of the result")
+
+# List result arguments
+list_results_args = cmd2.Cmd2ArgumentParser(
+    description="List all results.",
+    epilog='Example:\n list_results -f "my-runner"',
+)
+
+list_results_args.add_argument(
+    "-f",
+    "--find",
+    type=str,
+    help="Optional field to find result(s) with keyword",
+    nargs="?",
+)
+
+list_results_args.add_argument(
+    "-p",
+    "--pagination",
+    type=str,
+    help="Optional tuple to paginate result(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
+    nargs="?",
+)

moonshot/integrations/cli/benchmark/run.py

@@ -1,3 +1,5 @@
+from ast import literal_eval
+
 import cmd2
 from rich.console import Console
 from rich.table import Table
@@ -7,6 +9,7 @@ from moonshot.integrations.cli.common.display_helper import (
     display_view_list_format,
     display_view_str_format,
 )
+from moonshot.integrations.cli.utils.process_data import filter_data
 
 console = Console()
 
@@ -14,21 +17,38 @@ console = Console()
 # ------------------------------------------------------------------------------
 # CLI Functions
 # ------------------------------------------------------------------------------
-def list_runs() -> None:
+def list_runs(args) -> list | None:
     """
     List all runs.
 
     This function retrieves all available runs by calling the api_get_all_run function from the
-    moonshot.api module. It then calls the display_runs function to present the retrieved run information
+    moonshot.api module. It then calls the _display_runs function to present the retrieved run information
     in a user-friendly format on the command line interface. If an exception occurs during the retrieval
     or display process, it prints an error message.
 
+    Args:
+        args: A namespace object from argparse. It should have an optional attribute:
+        find (str): Optional field to find run(s) with a keyword.
+        pagination (str): Optional field to paginate runs.
+
     Returns:
-        None
+        list | None: A list of Run or None if there is no result.
     """
+
     try:
         runner_run_info = api_get_all_run()
-        display_runs(runner_run_info)
+        keyword = args.find.lower() if args.find else ""
+        pagination = literal_eval(args.pagination) if args.pagination else ()
+
+        if runner_run_info:
+            filtered_runs_list = filter_data(runner_run_info, keyword, pagination)
+            if filtered_runs_list:
+                _display_runs(filtered_runs_list)
+                return filtered_runs_list
+
+        console.print("[red]There are no runs found.[/red]")
+        return None
+
     except Exception as e:
         print(f"[list_runs]: {str(e)}")
 
@@ -38,7 +58,7 @@ def view_run(args) -> None:
     View the details of a specific run.
 
     This function retrieves and displays information about a specific run associated with a runner. It uses the runner
-    identifier provided in the arguments to fetch the data and then calls the display_runs function to present it in a
+    identifier provided in the arguments to fetch the data and then calls the _display_runs function to present it in a
     user-friendly format.
 
     Args:
@@ -50,7 +70,7 @@ def view_run(args) -> None:
     """
     try:
         runner_run_info = api_get_all_run(args.runner_id)
-        display_runs(runner_run_info)
+        _display_runs(runner_run_info)
     except Exception as e:
         print(f"[view_run]: {str(e)}")
 
@@ -58,7 +78,7 @@ def view_run(args) -> None:
 # ------------------------------------------------------------------------------
 # Helper functions: Display on cli
 # ------------------------------------------------------------------------------
-def display_runs(runs_list: list):
+def _display_runs(runs_list: list):
     """
     Display a list of runs in a table format.
 
@@ -74,59 +94,56 @@ def display_runs(runs_list: list):
     Returns:
         None
     """
-    if runs_list:
-        table = Table(
-            title="List of Runs", show_lines=True, expand=True, header_style="bold"
+    table = Table(
+        title="List of Runs", show_lines=True, expand=True, header_style="bold"
+    )
+    table.add_column("No.", width=2)
+    table.add_column("Run", justify="left", width=78)
+    table.add_column("Contains", justify="left", width=20, overflow="fold")
+    for idx, run in enumerate(runs_list, 1):
+        (
+            run_id,
+            runner_id,
+            runner_type,
+            runner_args,
+            endpoints,
+            results_file,
+            start_time,
+            end_time,
+            duration,
+            error_messages,
+            raw_results,
+            results,
+            status,
+            *other_args,
+        ) = run.values()
+
+        duration_info = f"[blue]Period:[/blue] {start_time} - {end_time} ({duration}s)"
+        run_id = display_view_str_format("Run ID", run_id)
+        runner_id = display_view_str_format("Runner ID", runner_id)
+        runner_type = display_view_str_format("Runner Type", runner_type)
+        runner_args = display_view_str_format("Runner Args", runner_args)
+        status_info = display_view_str_format("Status", status)
+        results_info = display_view_str_format("Results File", results_file)
+        endpoints_info = display_view_list_format("Endpoints", endpoints)
+        error_messages_info = display_view_list_format("Error Messages", error_messages)
+
+        has_raw_results = bool(raw_results)
+        has_results = bool(results)
+        idx = run.get("idx", idx)
+
+        result_info = (
+            f"[red]{runner_id}[/red]\n\n{run_id}\n\n{duration_info}\n\n{status_info}"
+        )
+        contains_info = (
+            f"{results_info}\n\n{error_messages_info}\n\n{endpoints_info}\n\n"
+            f"[blue]Has Raw Results: {has_raw_results}[/blue]\n\n"
+            f"[blue]Has Results: {has_results}[/blue]"
         )
-        table.add_column("No.", width=2)
-        table.add_column("Run", justify="left", width=78)
-        table.add_column("Contains", justify="left", width=20, overflow="fold")
-        for run_number, run in enumerate(runs_list, 1):
-            (
-                run_id,
-                runner_id,
-                runner_type,
-                runner_args,
-                endpoints,
-                results_file,
-                start_time,
-                end_time,
-                duration,
-                error_messages,
-                raw_results,
-                results,
-                status,
-            ) = run.values()
-
-            duration_info = (
-                f"[blue]Period:[/blue] {start_time} - {end_time} ({duration}s)"
-            )
-            run_id = display_view_str_format("Run ID", run_id)
-            runner_id = display_view_str_format("Runner ID", runner_id)
-            runner_type = display_view_str_format("Runner Type", runner_type)
-            runner_args = display_view_str_format("Runner Args", runner_args)
-            status_info = display_view_str_format("Status", status)
-            results_info = display_view_str_format("Results File", results_file)
-            endpoints_info = display_view_list_format("Endpoints", endpoints)
-            error_messages_info = display_view_list_format(
-                "Error Messages", error_messages
-            )
-
-            has_raw_results = bool(raw_results)
-            has_results = bool(results)
-
-            result_info = f"[red]{runner_id}[/red]\n\n{run_id}\n\n{duration_info}\n\n{status_info}"
-            contains_info = (
-                f"{results_info}\n\n{error_messages_info}\n\n{endpoints_info}\n\n"
-                f"[blue]Has Raw Results: {has_raw_results}[/blue]\n\n"
-                f"[blue]Has Results: {has_results}[/blue]"
-            )
-
-            table.add_section()
-            table.add_row(str(run_number), result_info, contains_info)
-        console.print(table)
-    else:
-        console.print("[red]There are no results found.[/red]")
+
+        table.add_section()
+        table.add_row(str(idx), result_info, contains_info)
+    console.print(table)
 
 
 # ------------------------------------------------------------------------------
@@ -138,3 +155,26 @@ view_run_args = cmd2.Cmd2ArgumentParser(
     epilog="Example:\n view_run my-new-cookbook-runner",
 )
 view_run_args.add_argument("runner_id", type=str, help="Name of the runner")
+
+
+# List run arguments
+list_runs_args = cmd2.Cmd2ArgumentParser(
+    description="List all runs.",
+    epilog='Example:\n list_runs -f "my-run"',
+)
+
+list_runs_args.add_argument(
+    "-f",
+    "--find",
+    type=str,
+    help="Optional field to find run(s) with keyword",
+    nargs="?",
+)
+
+list_runs_args.add_argument(
+    "-p",
+    "--pagination",
+    type=str,
+    help="Optional tuple to paginate run(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
+    nargs="?",
+)