aiverify-moonshot 0.4.0__py3-none-any.whl

moonshot/integrations/cli/benchmark/result.py

@@ -0,0 +1,216 @@
+import cmd2
+from rich.console import Console
+from rich.table import Table
+
+from moonshot.api import api_delete_result, api_get_all_result, api_read_result
+from moonshot.integrations.cli.benchmark.cookbook import show_cookbook_results
+from moonshot.integrations.cli.benchmark.recipe import show_recipe_results
+from moonshot.integrations.cli.common.display_helper import (
+    display_view_list_format,
+    display_view_str_format,
+)
+
+console = Console()
+
+
+# ------------------------------------------------------------------------------
+# CLI Functions
+# ------------------------------------------------------------------------------
+def list_results() -> None:
+    """
+    List all available results.
+
+    This function retrieves all available results by calling the api_get_all_result_name function from the
+    moonshot.api module. It then creates a table with the result id and name. If there are no results, it prints a
+    message indicating that no results were found.
+
+    Returns:
+        None
+    """
+    try:
+        results_list = api_get_all_result()
+        display_results(results_list)
+    except Exception as e:
+        print(f"[list_results]: {str(e)}")
+
+
+def view_result(args) -> None:
+    """
+    View a specific result.
+
+    This function retrieves a specific result by calling the api_read_result function from the
+    moonshot.api module using the result filename provided in the args.
+    It then checks if the result filename starts with "cookbook". If it does, it displays the result using the
+    display_view_cookbook_result function. Otherwise, it uses the display_view_recipe_result function.
+
+    Args:
+        args: A namespace object from argparse. It should have the following attribute:
+            result_filename (str): The filename of the result to view.
+
+    Returns:
+        None
+    """
+    try:
+        result_info = api_read_result(args.result_filename)
+        if result_info["metadata"].get("cookbooks"):
+            display_view_cookbook_result(result_info)
+        elif result_info["metadata"].get("recipes"):
+            display_view_recipe_result(result_info)
+        else:
+            print("[view_result]: Unable to determine cookbook or recipe")
+    except Exception as e:
+        print(f"[view_result]: {str(e)}")
+
+
+def delete_result(args) -> None:
+    """
+    Delete a result.
+
+    This function deletes a result with the specified identifier. It prompts the user for confirmation before proceeding
+    with the deletion. If the user confirms, it calls the api_delete_result function from the moonshot.api module to
+    delete the result. If the deletion is successful, it prints a confirmation message. If an exception occurs, it
+    prints an error message.
+
+    Args:
+        args: A namespace object from argparse. It should have the following attribute:
+            result (str): The identifier of the result to delete.
+
+    Returns:
+        None
+    """
+    # Confirm with the user before deleting a result
+    confirmation = console.input(
+        "[bold red]Are you sure you want to delete the result (y/N)? [/]"
+    )
+    if confirmation.lower() != "y":
+        console.print("[bold yellow]Result deletion cancelled.[/]")
+        return
+    try:
+        api_delete_result(args.result)
+        print("[delete_result]: Result deleted.")
+    except Exception as e:
+        print(f"[delete_result]: {str(e)}")
+
+
+# ------------------------------------------------------------------------------
+# Helper functions: Display on cli
+# ------------------------------------------------------------------------------
+def display_results(results_list):
+    """
+    Display a list of results.
+
+    This function takes a list of results and displays them in a table format. If the list is empty, it prints a
+    message indicating that no results were found.
+
+    Args:
+        results_list (list): A list of results. Each result is a dictionary with keys 'id' and 'name'.
+
+    Returns:
+        None
+    """
+    if results_list:
+        table = Table(
+            title="List of Results", show_lines=True, expand=True, header_style="bold"
+        )
+        table.add_column("No.", width=2)
+        table.add_column("Result", justify="left", width=78)
+        table.add_column("Contains", justify="left", width=20, overflow="fold")
+        for result_id, result in enumerate(results_list, 1):
+            metadata, results = result.values()
+
+            id = metadata["id"]
+            start_time = metadata["start_time"]
+            end_time = metadata["end_time"]
+            duration = metadata["duration"]
+            status = metadata["status"]
+            recipes = metadata["recipes"]
+            cookbooks = metadata["cookbooks"]
+            endpoints = metadata["endpoints"]
+            num_of_prompts = metadata["num_of_prompts"]
+            random_seed = metadata["random_seed"]
+            system_prompt = metadata["system_prompt"]
+
+            duration_info = (
+                f"[blue]Period:[/blue] {start_time} - {end_time} ({duration}s)"
+            )
+            status_info = display_view_str_format("Status", status)
+            recipes_info = display_view_list_format("Recipes", recipes)
+            cookbooks_info = display_view_list_format("Cookbooks", cookbooks)
+            endpoints_info = display_view_list_format("Endpoints", endpoints)
+            prompts_info = display_view_str_format("Number of Prompts", num_of_prompts)
+            seed_info = display_view_str_format("Seed", random_seed)
+            system_prompt_info = display_view_str_format("System Prompt", system_prompt)
+
+            result_info = f"[red]id: {id}[/red]\n\n{duration_info}\n\n{status_info}"
+            contains_info = (
+                f"{recipes_info}\n\n{cookbooks_info}\n\n{endpoints_info}\n\n{prompts_info}"
+                f"\n\n{seed_info}\n\n{system_prompt_info}"
+            )
+
+            table.add_section()
+            table.add_row(str(result_id), result_info, contains_info)
+        console.print(table)
+    else:
+        console.print("[red]There are no results found.[/red]")
+
+
+def display_view_recipe_result(result_info):
+    """
+    Display the recipe result.
+
+    This function takes the result file and result info as arguments. It converts the result info into a dictionary
+    using the convert_string_tuples_in_dict function. It then retrieves the recipes, endpoints, and duration from the
+    converted result info. Finally, it calls the show_recipe_results function from the
+    moonshot.integrations.cli.benchmark.recipe module to display the recipe results.
+
+    Args:
+        result_info (dict): The result info.
+
+    Returns:
+        None
+    """
+    recipes = result_info["metadata"]["recipes"]
+    endpoints = result_info["metadata"]["endpoints"]
+    duration = result_info["metadata"]["duration"]
+    show_recipe_results(recipes, endpoints, result_info, duration)
+
+
+def display_view_cookbook_result(result_info):
+    """
+    Display the cookbook result.
+
+    This function takes the result file and result info as arguments. It converts the result info into a dictionary
+    using the convert_string_tuples_in_dict function. It then retrieves the cookbooks, endpoints, and duration from the
+    converted result info. Finally, it calls the show_cookbook_results function from the
+    moonshot.integrations.cli.benchmark.cookbook module to display the cookbook results.
+
+    Args:
+        result_info (dict): The result info.
+
+    Returns:
+        None
+    """
+    cookbooks = result_info["metadata"]["cookbooks"]
+    endpoints = result_info["metadata"]["endpoints"]
+    duration = result_info["metadata"]["duration"]
+    show_cookbook_results(cookbooks, endpoints, result_info, duration)
+
+
+# ------------------------------------------------------------------------------
+# Cmd2 Arguments Parsers
+# ------------------------------------------------------------------------------
+# View result arguments
+view_result_args = cmd2.Cmd2ArgumentParser(
+    description="View a result file.",
+    epilog="Example:\n view_result my-new-cookbook-runner",
+)
+view_result_args.add_argument(
+    "result_filename", type=str, help="Name of the result file"
+)
+
+# Delete result arguments
+delete_result_args = cmd2.Cmd2ArgumentParser(
+    description="Delete a result.",
+    epilog="Example:\n delete_result my-new-cookbook-runner",
+)
+delete_result_args.add_argument("result", type=str, help="Name of the result")

moonshot/integrations/cli/benchmark/run.py

@@ -0,0 +1,140 @@
+import cmd2
+from rich.console import Console
+from rich.table import Table
+
+from moonshot.api import api_get_all_run
+from moonshot.integrations.cli.common.display_helper import (
+    display_view_list_format,
+    display_view_str_format,
+)
+
+console = Console()
+
+
+# ------------------------------------------------------------------------------
+# CLI Functions
+# ------------------------------------------------------------------------------
+def list_runs() -> None:
+    """
+    List all runs.
+
+    This function retrieves all available runs by calling the api_get_all_run function from the
+    moonshot.api module. It then calls the display_runs function to present the retrieved run information
+    in a user-friendly format on the command line interface. If an exception occurs during the retrieval
+    or display process, it prints an error message.
+
+    Returns:
+        None
+    """
+    try:
+        runner_run_info = api_get_all_run()
+        display_runs(runner_run_info)
+    except Exception as e:
+        print(f"[list_runs]: {str(e)}")
+
+
+def view_run(args) -> None:
+    """
+    View the details of a specific run.
+
+    This function retrieves and displays information about a specific run associated with a runner. It uses the runner
+    identifier provided in the arguments to fetch the data and then calls the display_runs function to present it in a
+    user-friendly format.
+
+    Args:
+        args: A namespace object from argparse. It should have the following attribute:
+            runner (str): The identifier of the runner whose runs are to be viewed.
+
+    Returns:
+        None
+    """
+    try:
+        runner_run_info = api_get_all_run(args.runner_id)
+        display_runs(runner_run_info)
+    except Exception as e:
+        print(f"[view_run]: {str(e)}")
+
+
+# ------------------------------------------------------------------------------
+# Helper functions: Display on cli
+# ------------------------------------------------------------------------------
+def display_runs(runs_list: list):
+    """
+    Display a list of runs in a table format.
+
+    This function takes a list of run information and displays it in a table format using the rich library's
+    Table object.
+
+    Each run's details are formatted and added as a row in the table.
+    If there are no runs to display, a message is printed to indicate that no results were found.
+
+    Args:
+        runs_list (list): A list of dictionaries, where each dictionary contains details of a run.
+
+    Returns:
+        None
+    """
+    if runs_list:
+        table = Table(
+            title="List of Runs", show_lines=True, expand=True, header_style="bold"
+        )
+        table.add_column("No.", width=2)
+        table.add_column("Run", justify="left", width=78)
+        table.add_column("Contains", justify="left", width=20, overflow="fold")
+        for run_number, run in enumerate(runs_list, 1):
+            (
+                run_id,
+                runner_id,
+                runner_type,
+                runner_args,
+                endpoints,
+                results_file,
+                start_time,
+                end_time,
+                duration,
+                error_messages,
+                raw_results,
+                results,
+                status,
+            ) = run.values()
+
+            duration_info = (
+                f"[blue]Period:[/blue] {start_time} - {end_time} ({duration}s)"
+            )
+            run_id = display_view_str_format("Run ID", run_id)
+            runner_id = display_view_str_format("Runner ID", runner_id)
+            runner_type = display_view_str_format("Runner Type", runner_type)
+            runner_args = display_view_str_format("Runner Args", runner_args)
+            status_info = display_view_str_format("Status", status)
+            results_info = display_view_str_format("Results File", results_file)
+            endpoints_info = display_view_list_format("Endpoints", endpoints)
+            error_messages_info = display_view_list_format(
+                "Error Messages", error_messages
+            )
+
+            has_raw_results = bool(raw_results)
+            has_results = bool(results)
+
+            result_info = f"[red]{runner_id}[/red]\n\n{run_id}\n\n{duration_info}\n\n{status_info}"
+            contains_info = (
+                f"{results_info}\n\n{error_messages_info}\n\n{endpoints_info}\n\n"
+                f"[blue]Has Raw Results: {has_raw_results}[/blue]\n\n"
+                f"[blue]Has Results: {has_results}[/blue]"
+            )
+
+            table.add_section()
+            table.add_row(str(run_number), result_info, contains_info)
+        console.print(table)
+    else:
+        console.print("[red]There are no results found.[/red]")
+
+
+# ------------------------------------------------------------------------------
+# Cmd2 Arguments Parsers
+# ------------------------------------------------------------------------------
+# View run arguments
+view_run_args = cmd2.Cmd2ArgumentParser(
+    description="View a runner runs.",
+    epilog="Example:\n view_run my-new-cookbook-runner",
+)
+view_run_args.add_argument("runner_id", type=str, help="Name of the runner")

moonshot/integrations/cli/benchmark/runner.py

@@ -0,0 +1,174 @@
+import cmd2
+from rich.console import Console
+from rich.table import Table
+
+from moonshot.api import (
+    api_delete_runner,
+    api_get_all_run,
+    api_get_all_runner,
+    api_get_available_session_info,
+    api_load_session,
+    api_read_runner,
+)
+from moonshot.integrations.cli.common.display_helper import (
+    display_view_list_format,
+    display_view_str_format,
+)
+
+console = Console()
+
+
+# ------------------------------------------------------------------------------
+# CLI Functions
+# ------------------------------------------------------------------------------
+def list_runners() -> None:
+    """
+    List all runners.
+
+    This function retrieves and displays information about all runners, including their associated runs and session
+    information. It fetches the data using the api_get_all_runner, api_get_all_run, and api_get_available_session_info
+    functions, then calls the display_runners function to present it in a user-friendly format.
+
+    Returns:
+        None
+    """
+    try:
+        runner_info = api_get_all_runner()
+        runner_run_info = api_get_all_run()
+        _, runner_session_info = api_get_available_session_info()
+        display_runners(runner_info, runner_run_info, runner_session_info)
+    except Exception as e:
+        print(f"[list_runners]: {str(e)}")
+
+
+def view_runner(args) -> None:
+    """
+    View a specific runner.
+
+    This function retrieves and displays information about a specific runner, including its associated runs and session
+    information. It uses the runner identifier provided in the arguments to fetch the data and then calls the
+    display_runners function to present it in a user-friendly format.
+
+    Args:
+        args: A namespace object from argparse. It should have the following attribute:
+            runner (str): The identifier of the runner to view.
+
+    Returns:
+        None
+    """
+    try:
+        runner_info = api_read_runner(args.runner)
+        runner_run_info = api_get_all_run(args.runner)
+        runner_session_info = api_load_session(args.runner)
+        display_runners([runner_info], runner_run_info, [runner_session_info])
+    except Exception as e:
+        print(f"[view_runner]: {str(e)}")
+
+
+def delete_runner(args) -> None:
+    """
+    Delete a runner.
+
+    This function deletes a runner with the specified identifier. It prompts the user for confirmation before proceeding
+    with the deletion. If the user confirms, it calls the api_delete_runner function from the moonshot.api module to
+    delete the runner. If the deletion is successful, it prints a confirmation message. If an exception occurs, it
+    prints an error message.
+
+    Args:
+        args: A namespace object from argparse. It should have the following attribute:
+            runner (str): The identifier of the runner to delete.
+
+    Returns:
+        None
+    """
+    # Confirm with the user before deleting a runner
+    confirmation = console.input(
+        "[bold red]Are you sure you want to delete the runner (y/N)? [/]"
+    )
+    if confirmation.lower() != "y":
+        console.print("[bold yellow]Runner deletion cancelled.[/]")
+        return
+    try:
+        api_delete_runner(args.runner)
+        print("[delete_runner]: Runner deleted.")
+    except Exception as e:
+        print(f"[delete_runner]: {str(e)}")
+
+
+# ------------------------------------------------------------------------------
+# Helper functions: Display on cli
+# ------------------------------------------------------------------------------
+def display_runners(
+    runner_list: list, runner_run_info_list: list, runner_session_info_list: list
+) -> None:
+    """
+    Display runners in a table format.
+
+    This function takes lists of runner information, run information, and session information, then displays them in a
+    table format on the command line interface. Each runner is listed with details such as the runner's ID, name,
+    description, number of runs, number of sessions, database file, and endpoints.
+
+    Args:
+        runner_list: A list of dictionaries, where each dictionary contains information about a runner.
+
+        runner_run_info_list: A list of dictionaries, where each dictionary contains information about a run
+        associated with a runner.
+
+        runner_session_info_list: A list of dictionaries, where each dictionary contains information about a session
+        associated with a runner.
+
+    Returns:
+        None
+    """
+    if runner_list:
+        table = Table(
+            title="List of Runners", show_lines=True, expand=True, header_style="bold"
+        )
+        table.add_column("No.", width=2)
+        table.add_column("Runner", justify="left", width=78)
+        table.add_column("Contains", justify="left", width=20, overflow="fold")
+        for runner_id, runner in enumerate(runner_list, 1):
+            (id, name, db_file, endpoints, description) = runner.values()
+
+            db_info = display_view_str_format("Database", db_file)
+            endpoints_info = display_view_list_format("Endpoints", endpoints)
+
+            runs_count = sum(
+                run_info["runner_id"] == id for run_info in runner_run_info_list
+            )
+            # Handle the case where session_info can be None
+            sessions_count = sum(
+                session_info is not None and session_info["session_id"] == id
+                for session_info in runner_session_info_list
+            )
+
+            runner_info = (
+                f"[red]id: {id}[/red]\n\n[blue]{name}[/blue]\n{description}\n"
+                f"[blue]Number of Runs:[/blue] {runs_count}\n"
+                f"[blue]Number of Sessions:[/blue] {sessions_count}"
+            )
+            contains_info = f"{db_info}\n\n{endpoints_info}"
+
+            table.add_section()
+            table.add_row(str(runner_id), runner_info, contains_info)
+        console.print(table)
+    else:
+        console.print("[red]There are no runners found.[/red]")
+
+
+# ------------------------------------------------------------------------------
+# Cmd2 Arguments Parsers
+# ------------------------------------------------------------------------------
+# View runner arguments
+view_runner_args = cmd2.Cmd2ArgumentParser(
+    description="View a runner.",
+    epilog="Example:\n view_runner my-new-cookbook-runner",
+)
+view_runner_args.add_argument("runner", type=str, help="Name of the runner")
+
+# Delete runner arguments
+delete_runner_args = cmd2.Cmd2ArgumentParser(
+    description="Delete a runner.",
+    epilog="Example:\n delete_runner my-new-cookbook-runner",
+)
+delete_runner_args.add_argument("runner", type=str, help="Name of the runner")

moonshot/integrations/cli/cli.py

@@ -0,0 +1,64 @@
+import cmd2
+from cmd2 import Statement
+from rich.console import Console
+
+from moonshot.integrations.cli.active_session_cfg import active_session
+from moonshot.integrations.cli.benchmark.benchmark import BenchmarkCommandSet
+from moonshot.integrations.cli.common.common import CommonCommandSet
+from moonshot.integrations.cli.initialisation.initialisation import (
+    InitialisationCommandSet,
+)
+from moonshot.integrations.cli.redteam.redteam import RedTeamCommandSet
+from moonshot.integrations.cli.redteam.session import (
+    manual_red_teaming,
+    update_chat_display,
+)
+
+console = Console()
+
+
+class CommandLineInterface(cmd2.Cmd):
+    def __init__(self):
+        super().__init__(terminators=[])
+        self.prompt = "moonshot > "
+        self.welcome()
+
+    def welcome(self) -> None:
+        """
+        Display Project Moonshot logo
+        """
+        logo = "  _____           _           _     __  __                       _           _   \n"
+        logo += " |  __ \\         (_)         | |   |  \\/  |                     | |         | |  \n"
+        logo += " | |__) | __ ___  _  ___  ___| |_  | \\  / | ___   ___  _ __  ___| |__   ___ | |_ \n"
+        logo += " |  ___/ '__/ _ \\| |/ _ \\/ __| __| | |\\/| |/ _ \\ / _ \\| '_ \\/ __| '_ \\ / _ \\| __|\n"
+        logo += " | |   | | | (_) | |  __/ (__| |_  | |  | | (_) | (_) | | | \\__ \\ | | | (_) | |_ \n"
+        logo += " |_|   |_|  \\___/| |\\___|\\___|\\__| |_|  |_|\\___/ \\___/|_| |_|___/_| |_|\\___/ \\__|\n"
+        logo += "                _/ |                                                             \n"
+        logo += "               |__/                                                              \n"
+        logo += "\n"
+        print(logo)
+
+    def default(self, statement: Statement) -> None:
+        # if there is an active session, anything entered other than recognised commands
+        # will be considered prompts
+        if active_session:
+            user_prompt = statement.command + " " + statement
+            user_prompt = user_prompt.strip()
+            manual_red_teaming(user_prompt)
+            # Update chat display with response
+            update_chat_display()
+
+    def postcmd(self, stop, line):
+        if active_session:
+            if active_session["context_strategy"]:
+                cs_prompt = f"CS: ({active_session['context_strategy']},{active_session['cs_num_of_prev_prompts']})] > "
+            else:
+                cs_prompt = f"CS: {active_session['context_strategy']}]> "
+            self.prompt = (
+                f"moonshot ({active_session['session_id']}) "
+                f"[PT: {active_session['prompt_template']}, "
+                f"{cs_prompt}"
+            )
+        else:
+            self.prompt = "moonshot > "
+        return stop

moonshot/integrations/cli/common/common.py

@@ -0,0 +1,72 @@
+import argparse
+
+import cmd2
+
+from moonshot.integrations.cli.common.connectors import (
+    add_endpoint,
+    add_endpoint_args,
+    delete_endpoint,
+    delete_endpoint_args,
+    list_connector_types,
+    list_endpoints,
+    update_endpoint,
+    update_endpoint_args,
+    view_endpoint,
+    view_endpoint_args,
+)
+from moonshot.integrations.cli.common.prompt_template import (
+    delete_prompt_template,
+    delete_prompt_template_args,
+    list_prompt_templates,
+)
+
+
+@cmd2.with_default_category("Moonshot Common")
+class CommonCommandSet(cmd2.CommandSet):
+    def __init__(self):
+        super().__init__()
+
+    # ------------------------------------------------------------------------------
+    # List contents
+    # ------------------------------------------------------------------------------
+
+    def do_list_connector_types(self, _: cmd2.Statement) -> None:
+        list_connector_types()
+
+    def do_list_endpoints(self, _: cmd2.Statement) -> None:
+        list_endpoints()
+
+    def do_list_prompt_templates(self, _: cmd2.Statement) -> None:
+        list_prompt_templates()
+
+    @cmd2.with_argparser(delete_prompt_template_args)
+    def do_delete_prompt_template(self, args: argparse.Namespace) -> None:
+        delete_prompt_template(args)
+
+    # ------------------------------------------------------------------------------
+    # Add contents
+    # ------------------------------------------------------------------------------
+    @cmd2.with_argparser(add_endpoint_args)
+    def do_add_endpoint(self, args: argparse.Namespace) -> None:
+        add_endpoint(args)
+
+    # ------------------------------------------------------------------------------
+    # Delete contents
+    # ------------------------------------------------------------------------------
+    @cmd2.with_argparser(delete_endpoint_args)
+    def do_delete_endpoint(self, args: argparse.Namespace) -> None:
+        delete_endpoint(args)
+
+    # ------------------------------------------------------------------------------
+    # Update contents
+    # ------------------------------------------------------------------------------
+    @cmd2.with_argparser(update_endpoint_args)
+    def do_update_endpoint(self, args: argparse.Namespace) -> None:
+        update_endpoint(args)
+
+    # ------------------------------------------------------------------------------
+    # View contents
+    # ------------------------------------------------------------------------------
+    @cmd2.with_argparser(view_endpoint_args)
+    def do_view_endpoint(self, args: argparse.Namespace) -> None:
+        view_endpoint(args)