aiverify-moonshot 0.4.2__tar.gz → 0.4.3__tar.gz

{aiverify_moonshot-0.4.2 → aiverify_moonshot-0.4.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: aiverify-moonshot
-Version: 0.4.2
+Version: 0.4.3
 Summary: AI Verify advances Gen AI testing with Project Moonshot.
 Project-URL: Repository, https://github.com/aiverify-foundation/moonshot
 Project-URL: Documentation, https://aiverify-foundation.github.io/moonshot/
@@ -43,7 +43,7 @@ Description-Content-Type: text/markdown
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.2**
+**Version 0.4.3**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.2 → aiverify_moonshot-0.4.3}/README.md

@@ -2,7 +2,7 @@
 
 ![Moonshot Logo](https://github.com/aiverify-foundation/moonshot/raw/main/misc/aiverify-moonshot-logo.png)
 
-**Version 0.4.2**
+**Version 0.4.3**
 
 A simple and modular tool to evaluate any LLM application.
 

{aiverify_moonshot-0.4.2 → aiverify_moonshot-0.4.3}/moonshot/api.py

@@ -29,6 +29,7 @@ from moonshot.src.api.api_dataset import (
     api_delete_dataset,
     api_get_all_datasets,
     api_get_all_datasets_name,
+    api_create_datasets
 )
 from moonshot.src.api.api_environment_variables import api_set_environment_variables
 from moonshot.src.api.api_metrics import (
@@ -115,6 +116,7 @@ __all__ = [
     "api_read_cookbook",
     "api_read_cookbooks",
     "api_update_cookbook",
+    "api_create_datasets",
     "api_delete_dataset",
     "api_get_all_datasets",
     "api_get_all_datasets_name",

{aiverify_moonshot-0.4.2 → aiverify_moonshot-0.4.3}/moonshot/integrations/cli/benchmark/cookbook.py

@@ -23,7 +23,7 @@ from moonshot.integrations.cli.benchmark.recipe import (
     display_view_statistics_format,
 )
 from moonshot.integrations.cli.common.display_helper import display_view_list_format
-from moonshot.src.utils.find_feature import find_keyword
+from moonshot.integrations.cli.utils.process_data import filter_data
 
 console = Console()
 
@@ -62,29 +62,31 @@ def list_cookbooks(args) -> list | None:
 
     This function retrieves all available cookbooks by calling the api_get_all_cookbook function from the
     moonshot.api module.
-    It then displays the retrieved cookbooks using the display_cookbooks function.
+    It then displays the retrieved cookbooks using the _display_cookbooks function.
 
     Args:
         args: A namespace object from argparse. It should have an optional attribute:
         find (str): Optional field to find cookbook(s) with a keyword.
+        pagination (str): Optional field to paginate cookbooks.
 
     Returns:
         list | None: A list of Cookbook or None if there is no result.
     """
+
     try:
         cookbooks_list = api_get_all_cookbook()
         keyword = args.find.lower() if args.find else ""
-        if keyword:
-            filtered_cookbooks_list = find_keyword(keyword, cookbooks_list)
+        pagination = literal_eval(args.pagination) if args.pagination else ()
+
+        if cookbooks_list:
+            filtered_cookbooks_list = filter_data(cookbooks_list, keyword, pagination)
             if filtered_cookbooks_list:
-                display_cookbooks(filtered_cookbooks_list)
+                _display_cookbooks(filtered_cookbooks_list)
                 return filtered_cookbooks_list
-            else:
-                print("No cookbooks containing keyword found.")
-                return None
-        else:
-            display_cookbooks(cookbooks_list)
-            return cookbooks_list
+
+        console.print("[red]There are no cookbooks found.[/red]")
+        return None
+
     except Exception as e:
         print(f"[list_cookbooks]: {str(e)}")
 
@@ -236,7 +238,7 @@ def delete_cookbook(args) -> None:
 # ------------------------------------------------------------------------------
 # Helper functions: Display on cli
 # ------------------------------------------------------------------------------
-def display_cookbooks(cookbooks_list):
+def _display_cookbooks(cookbooks_list):
     """
     Display the list of cookbooks in a tabular format.
 
@@ -247,22 +249,20 @@ def display_cookbooks(cookbooks_list):
     Args:
         cookbooks_list (list): A list of dictionaries, where each dictionary contains the details of a cookbook.
     """
-    if cookbooks_list:
-        table = Table(
-            title="List of Cookbooks", show_lines=True, expand=True, header_style="bold"
-        )
-        table.add_column("No.", width=2)
-        table.add_column("Cookbook", justify="left", width=78)
-        table.add_column("Contains", justify="left", width=20, overflow="fold")
-        for cookbook_id, cookbook in enumerate(cookbooks_list, 1):
-            id, name, description, recipes = cookbook.values()
-            cookbook_info = f"[red]ID: {id}[/red]\n\n[blue]{name}[/blue]\n{description}"
-            recipes_info = display_view_list_format("Recipes", recipes)
-            table.add_section()
-            table.add_row(str(cookbook_id), cookbook_info, recipes_info)
-        console.print(table)
-    else:
-        console.print("[red]There are no cookbooks found.[/red]")
+    table = Table(
+        title="List of Cookbooks", show_lines=True, expand=True, header_style="bold"
+    )
+    table.add_column("No.", width=2)
+    table.add_column("Cookbook", justify="left", width=78)
+    table.add_column("Contains", justify="left", width=20, overflow="fold")
+    for idx, cookbook in enumerate(cookbooks_list, 1):
+        id, name, description, recipes, *other_args = cookbook.values()
+        idx = cookbook.get("idx", idx)
+        cookbook_info = f"[red]ID: {id}[/red]\n\n[blue]{name}[/blue]\n{description}"
+        recipes_info = display_view_list_format("Recipes", recipes)
+        table.add_section()
+        table.add_row(str(idx), cookbook_info, recipes_info)
+    console.print(table)
 
 
 def display_view_cookbook(cookbook_info):
@@ -573,3 +573,11 @@ list_cookbooks_args.add_argument(
     help="Optional field to find cookbook(s) with keyword",
     nargs="?",
 )
+
+list_cookbooks_args.add_argument(
+    "-p",
+    "--pagination",
+    type=str,
+    help="Optional tuple to paginate cookbook(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
+    nargs="?",
+)

{aiverify_moonshot-0.4.2 → aiverify_moonshot-0.4.3}/moonshot/integrations/cli/benchmark/datasets.py

@@ -1,3 +1,5 @@
+from ast import literal_eval
+
 import cmd2
 from rich.console import Console
 from rich.table import Table
@@ -8,7 +10,7 @@ from moonshot.api import (
     api_get_all_datasets_name,
 )
 from moonshot.integrations.cli.common.display_helper import display_view_str_format
-from moonshot.src.utils.find_feature import find_keyword
+from moonshot.integrations.cli.utils.process_data import filter_data
 
 console = Console()
 
@@ -21,12 +23,13 @@ def list_datasets(args) -> list | None:
     List all available datasets.
 
     This function retrieves all available datasets by calling the api_get_all_datasets function from the
-    moonshot.api module. It then displays the datasets using the display_datasets function. If an exception occurs,
+    moonshot.api module. It then displays the datasets using the _display_datasets function. If an exception occurs,
     it prints an error message.
 
     Args:
         args: A namespace object from argparse. It should have an optional attribute:
         find (str): Optional field to find dataset(s) with a keyword.
+        pagination (str): Optional field to paginate datasets.
 
     Returns:
         list | None: A list of Dataset or None if there is no result.
@@ -35,17 +38,16 @@ def list_datasets(args) -> list | None:
         print("Listing datasets may take a while...")
         datasets_list = api_get_all_datasets()
         keyword = args.find.lower() if args.find else ""
-        if keyword:
-            filtered_datasets_list = find_keyword(keyword, datasets_list)
+        pagination = literal_eval(args.pagination) if args.pagination else ()
+
+        if datasets_list:
+            filtered_datasets_list = filter_data(datasets_list, keyword, pagination)
             if filtered_datasets_list:
-                display_datasets(filtered_datasets_list)
+                _display_datasets(filtered_datasets_list)
                 return filtered_datasets_list
-            else:
-                print("No datasets containing keyword found.")
-                return None
-        else:
-            display_datasets(datasets_list)
-            return datasets_list
+
+        console.print("[red]There are no datasets found.[/red]")
+        return None
     except Exception as e:
         print(f"[list_datasets]: {str(e)}")
 
@@ -56,7 +58,7 @@ def view_dataset(args) -> None:
 
     This function retrieves all available datasets and their names by calling the api_get_all_datasets and
     api_get_all_datasets_name functions. It then finds the dataset with the name specified in args.dataset_filename
-    and displays it using the display_datasets function. If an exception occurs, it prints an error message.
+    and displays it using the _display_datasets function. If an exception occurs, it prints an error message.
 
     Args:
         args: A namespace object from argparse. It should have the following attribute:
@@ -72,8 +74,8 @@ def view_dataset(args) -> None:
 
         # Find the index of the dataset with the name args.dataset_filename
         dataset_index = datasets_name_list.index(args.dataset_filename)
-        # Pass the corresponding dataset from datasets_list to display_datasets
-        display_datasets([datasets_list[dataset_index]])
+        # Pass the corresponding dataset from datasets_list to _display_datasets
+        _display_datasets([datasets_list[dataset_index]])
 
     except Exception as e:
         print(f"[view_dataset]: {str(e)}")
@@ -112,7 +114,7 @@ def delete_dataset(args) -> None:
 # ------------------------------------------------------------------------------
 # Helper functions: Display on cli
 # ------------------------------------------------------------------------------
-def display_datasets(datasets_list: list):
+def _display_datasets(datasets_list: list):
     """
     Displays a list of datasets in a table format.
 
@@ -125,39 +127,38 @@ def display_datasets(datasets_list: list):
     Returns:
         None
     """
-    if datasets_list:
-        table = Table(
-            title="List of Datasets", show_lines=True, expand=True, header_style="bold"
+    table = Table(
+        title="List of Datasets", show_lines=True, expand=True, header_style="bold"
+    )
+    table.add_column("No.", width=2)
+    table.add_column("Dataset", justify="left", width=78)
+    for idx, dataset in enumerate(datasets_list, 1):
+        (
+            id,
+            name,
+            description,
+            _,
+            num_of_dataset_prompts,
+            created_date,
+            reference,
+            license,
+            *other_args,
+        ) = dataset.values()
+
+        idx = dataset.get("idx", idx)
+        prompt_info = display_view_str_format("Prompts", num_of_dataset_prompts)
+        created_date_info = display_view_str_format("Created Date", created_date)
+        license_info = display_view_str_format("License", license)
+        reference_info = display_view_str_format("Reference", reference)
+
+        dataset_info = (
+            f"[red]{id}[/red]\n\n[blue]{name}[/blue]\n{description}\n\n"
+            f"{prompt_info}\n\n{created_date_info}\n\n{license_info}\n\n{reference_info}"
         )
-        table.add_column("No.", width=2)
-        table.add_column("Dataset", justify="left", width=78)
-        for dataset_no, dataset in enumerate(datasets_list, 1):
-            (
-                id,
-                name,
-                description,
-                _,
-                num_of_dataset_prompts,
-                created_date,
-                reference,
-                license,
-            ) = dataset.values()
-
-            prompt_info = display_view_str_format("Prompts", num_of_dataset_prompts)
-            created_date_info = display_view_str_format("Created Date", created_date)
-            license_info = display_view_str_format("License", license)
-            reference_info = display_view_str_format("Reference", reference)
-
-            dataset_info = (
-                f"[red]{id}[/red]\n\n[blue]{name}[/blue]\n{description}\n\n"
-                f"{prompt_info}\n\n{created_date_info}\n\n{license_info}\n\n{reference_info}"
-            )
-
-            table.add_section()
-            table.add_row(str(dataset_no), dataset_info)
-        console.print(table)
-    else:
-        console.print("[red]There are no datasets found.[/red]")
+
+        table.add_section()
+        table.add_row(str(idx), dataset_info)
+    console.print(table)
 
 
 # ------------------------------------------------------------------------------
@@ -192,3 +193,11 @@ list_datasets_args.add_argument(
     help="Optional field to find dataset(s) with keyword",
     nargs="?",
 )
+
+list_datasets_args.add_argument(
+    "-p",
+    "--pagination",
+    type=str,
+    help="Optional tuple to paginate dataset(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
+    nargs="?",
+)

{aiverify_moonshot-0.4.2 → aiverify_moonshot-0.4.3}/moonshot/integrations/cli/benchmark/metrics.py

@@ -1,9 +1,11 @@
+from ast import literal_eval
+
 import cmd2
 from rich.console import Console
 from rich.table import Table
 
 from moonshot.api import api_delete_metric, api_get_all_metric, api_get_all_metric_name
-from moonshot.src.utils.find_feature import find_keyword
+from moonshot.integrations.cli.utils.process_data import filter_data
 
 console = Console()
 
@@ -16,31 +18,32 @@ def list_metrics(args) -> list | None:
     List all available metrics.
 
     This function retrieves all available metrics by calling the api_get_all_metric function from the
-    moonshot.api module. It then displays the metrics using the display_metrics function. If an exception occurs,
+    moonshot.api module. It then displays the metrics using the _display_metrics function. If an exception occurs,
     it prints an error message.
 
     Args:
         args: A namespace object from argparse. It should have an optional attribute:
         find (str): Optional field to find metric(s) with a keyword.
+        pagination (str): Optional field to paginate metrics.
 
     Returns:
         list | None: A list of Metric or None if there is no result.
     """
+
     try:
         print("Listing metrics may take a while...")
         metrics_list = api_get_all_metric()
         keyword = args.find.lower() if args.find else ""
-        if keyword:
-            filtered_metrics_list = find_keyword(keyword, metrics_list)
+        pagination = literal_eval(args.pagination) if args.pagination else ()
+
+        if metrics_list:
+            filtered_metrics_list = filter_data(metrics_list, keyword, pagination)
             if filtered_metrics_list:
-                display_metrics(filtered_metrics_list)
+                _display_metrics(filtered_metrics_list)
                 return filtered_metrics_list
-            else:
-                print("No metrics containing keyword found.")
-                return None
-        else:
-            display_metrics(metrics_list)
-            return metrics_list
+
+        console.print("[red]There are no metrics found.[/red]")
+        return None
     except Exception as e:
         print(f"[list_metrics]: {str(e)}")
 
@@ -51,7 +54,7 @@ def view_metric(args) -> None:
 
     This function retrieves all available metrics and their names by calling the api_get_all_metric and
     api_get_all_metric_name functions. It then finds the metric with the name specified in args.metric_filename
-    and displays it using the display_metrics function. If an exception occurs, it prints an error message.
+    and displays it using the _display_metrics function. If an exception occurs, it prints an error message.
 
     Args:
         args: A namespace object from argparse. It should have the following attribute:
@@ -67,8 +70,8 @@ def view_metric(args) -> None:
 
         # Find the index of the metric with the name args.metric_filename
         metric_index = metrics_name_list.index(args.metric_filename)
-        # Pass the corresponding metric from metrics_list to display_metrics
-        display_metrics([metrics_list[metric_index]])
+        # Pass the corresponding metric from metrics_list to _display_metrics
+        _display_metrics([metrics_list[metric_index]])
 
     except Exception as e:
         print(f"[view_metric]: {str(e)}")
@@ -107,7 +110,7 @@ def delete_metric(args) -> None:
 # ------------------------------------------------------------------------------
 # Helper functions: Display on cli
 # ------------------------------------------------------------------------------
-def display_metrics(metrics_list: list):
+def _display_metrics(metrics_list: list):
     """
     Displays a list of metrics in a table format.
 
@@ -120,21 +123,19 @@ def display_metrics(metrics_list: list):
     Returns:
         None
     """
-    if metrics_list:
-        table = Table(
-            title="List of Metrics", show_lines=True, expand=True, header_style="bold"
-        )
-        table.add_column("No.", width=2)
-        table.add_column("Metric", justify="left", width=78)
-        for metric_no, metric in enumerate(metrics_list, 1):
-            id, name, description = metric.values()
-            result_info = f"[red]id: {id}[/red]\n\n[blue]{name}[/blue]\n{description}"
-
-            table.add_section()
-            table.add_row(str(metric_no), result_info)
-        console.print(table)
-    else:
-        console.print("[red]There are no metrics found.[/red]")
+    table = Table(
+        title="List of Metrics", show_lines=True, expand=True, header_style="bold"
+    )
+    table.add_column("No.", width=2)
+    table.add_column("Metric", justify="left", width=78)
+    for idx, metric in enumerate(metrics_list, 1):
+        id, name, description, *other_args = metric.values()
+        idx = metric.get("idx", idx)
+        result_info = f"[red]id: {id}[/red]\n\n[blue]{name}[/blue]\n{description}"
+
+        table.add_section()
+        table.add_row(str(idx), result_info)
+    console.print(table)
 
 
 # ------------------------------------------------------------------------------
@@ -169,3 +170,11 @@ list_metrics_args.add_argument(
     help="Optional field to find metric(s) with keyword",
     nargs="?",
 )
+
+list_metrics_args.add_argument(
+    "-p",
+    "--pagination",
+    type=str,
+    help="Optional tuple to paginate metric(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
+    nargs="?",
+)