aiverify-moonshot 0.4.0__py3-none-any.whl

moonshot/src/api/api_dataset.py

@@ -0,0 +1,46 @@
+from pydantic import validate_call
+
+from moonshot.src.datasets.dataset import Dataset
+
+
+# ------------------------------------------------------------------------------
+# Datasets APIs
+# ------------------------------------------------------------------------------
+@validate_call
+def api_delete_dataset(ds_id: str) -> bool:
+    """
+    Deletes a dataset identified by its unique dataset ID.
+
+    Args:
+        ds_id (str): The unique identifier for the dataset to be deleted.
+
+    Returns:
+        bool: True if the dataset was successfully deleted.
+
+    Raises:
+        Exception: If the deletion process encounters an error.
+    """
+    return Dataset.delete(ds_id)
+
+
+def api_get_all_datasets() -> list[dict]:
+    """
+    This function retrieves all available datasets and returns them as a list of dictionaries. Each dictionary
+    represents a result and contains its information.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing a result.
+    """
+    _, datasets = Dataset.get_available_items()
+    return [dataset.to_dict() for dataset in datasets]
+
+
+def api_get_all_datasets_name() -> list[str]:
+    """
+    This function retrieves all available datasets names and returns them as a list.
+
+    Returns:
+        list[str]: A list of datasets names.
+    """
+    datasets_name, _ = Dataset.get_available_items()
+    return datasets_name

moonshot/src/api/api_environment_variables.py

@@ -0,0 +1,17 @@
+from moonshot.src.configs.env_variables import EnvironmentVars
+
+
+# ------------------------------------------------------------------------------
+# Environment Variables APIs
+# ------------------------------------------------------------------------------
+def api_set_environment_variables(env_vars: dict) -> None:
+    """
+    Sets the environment variables for the current session.
+
+    Args:
+        env_vars (dict): A dictionary containing the environment variables to set.
+
+    Returns:
+        None
+    """
+    EnvironmentVars.load_env(env_vars)

moonshot/src/api/api_metrics.py

@@ -0,0 +1,51 @@
+from pydantic import validate_call
+
+from moonshot.src.metrics.metric import Metric
+
+
+# ------------------------------------------------------------------------------
+# Metrics APIs
+# ------------------------------------------------------------------------------
+@validate_call
+def api_delete_metric(met_id: str) -> bool:
+    """
+    Deletes a metric identified by its unique metric ID.
+
+    Args:
+        met_id (str): The unique identifier for the metric to be deleted.
+
+    Returns:
+        bool: True if the metric was successfully deleted.
+
+    Raises:
+        Exception: If the deletion process encounters an error.
+    """
+    return Metric.delete(met_id)
+
+
+def api_get_all_metric() -> list[dict]:
+    """
+    Retrieves all available metrics.
+
+    This function calls the get_available_items method from the Metric class to retrieve all available metrics.
+    It then returns a list of dictionaries, each containing the details of a metric.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing a metric's details.
+    """
+    _, metrics_info = Metric.get_available_items()
+    return metrics_info
+
+
+def api_get_all_metric_name() -> list[str]:
+    """
+    Retrieves all available metric names.
+
+    This function calls the get_available_items method from the Metric class to retrieve all available metrics.
+    It then extracts the names of each metric and returns a list of these names.
+
+    Returns:
+        list[str]: A list of strings, each representing a metric name.
+    """
+    metrics_names, _ = Metric.get_available_items()
+    return metrics_names

moonshot/src/api/api_prompt_template.py

@@ -0,0 +1,43 @@
+from pydantic import validate_call
+
+from moonshot.src.prompt_templates.prompt_template import PromptTemplate
+
+
+# ------------------------------------------------------------------------------
+# Prompt Template APIs
+# ------------------------------------------------------------------------------
+def api_get_all_prompt_template_detail() -> list[dict]:
+    """
+    Retrieves all available prompt template details and returns them as a list of dictionaries.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing the details of a prompt template.
+    """
+    return PromptTemplate.get_all_prompt_template_details()
+
+
+def api_get_all_prompt_template_name() -> list[str]:
+    """
+    Retrieves all available prompt template names and returns them as a list.
+
+    Returns:
+        list[str]: A list of prompt template names.
+    """
+    return PromptTemplate.get_all_prompt_template_names()
+
+
+@validate_call
+def api_delete_prompt_template(pt_id: str) -> bool:
+    """
+    Deletes a prompt template by its identifier.
+
+    Args:
+        pt_id (str): The unique identifier of the prompt template to be deleted.
+
+    Returns:
+        bool: True if the prompt template was successfully deleted.
+
+    Raises:
+        Exception: If the deletion process encounters an error.
+    """
+    return PromptTemplate.delete(pt_id)

moonshot/src/api/api_recipe.py

@@ -0,0 +1,182 @@
+from pydantic import conlist, validate_call
+
+from moonshot.src.recipes.recipe import Recipe
+from moonshot.src.recipes.recipe_arguments import RecipeArguments
+
+
+# ------------------------------------------------------------------------------
+# Recipe APIs
+# ------------------------------------------------------------------------------
+@validate_call
+def api_create_recipe(
+    name: str,
+    description: str,
+    tags: list[str],
+    categories: list[str],
+    datasets: list[str],
+    prompt_templates: list[str],
+    metrics: list[str],
+    attack_modules: list[str],
+    grading_scale: dict[str, list[int]],
+) -> str:
+    """
+    Creates a new recipe with the given parameters.
+
+    This function takes various parameters that define a recipe, creates a RecipeArguments
+    object with these parameters, and then calls the Recipe.create method to create a new
+    recipe in the system.
+
+    Args:
+        name (str): The name of the recipe.
+        description (str): A description of the recipe.
+        tags (list[str]): A list of tags associated with the recipe.
+        categories (list[str]): A list of categories the recipe belongs to.
+        datasets (list[str]): A list of datasets used in the recipe.
+        prompt_templates (list[str]): A list of prompt templates for the recipe.
+        metrics (list[str]): A list of metrics to evaluate the recipe.
+        attack_modules (list[str]): A list of attack modules used in the recipe.
+        grading_scale (dict[str, list[int]]): A grading scale dictionary where the key is the grade and the
+        value is a list of integers representing the scale.
+
+    Returns:
+        str: The ID of the newly created recipe.
+    """
+    rec_args = RecipeArguments(
+        id="",
+        name=name,
+        description=description,
+        tags=tags,
+        categories=categories,
+        datasets=datasets,
+        prompt_templates=prompt_templates,
+        metrics=metrics,
+        attack_modules=attack_modules,
+        grading_scale=grading_scale,
+    )
+    return Recipe.create(rec_args)
+
+
+@validate_call
+def api_read_recipe(rec_id: str) -> dict:
+    """
+    Reads a recipe and returns its information.
+
+    This function takes a recipe ID as input, reads the corresponding recipe,
+    and returns a dictionary containing the recipe's information.
+
+    Args:
+        rec_id (str): The ID of the recipe.
+
+    Returns:
+        dict: A dictionary containing the recipe's information.
+    """
+    return Recipe.read(rec_id).to_dict()
+
+
+@validate_call
+def api_read_recipes(rec_ids: conlist(str, min_length=1)) -> list[dict]:
+    """
+    Reads multiple recipes and returns their information.
+
+    This function takes a list of recipe IDs as input, reads the corresponding recipes,
+    and returns a list of dictionaries containing each recipe's information.
+
+    Args:
+        rec_ids (list[str]): The IDs of the recipes.
+
+    Returns:
+        list[dict]: A list of dictionaries, each containing a recipe's information.
+    """
+    # This function uses list comprehension to iterate over the list of recipe IDs,
+    # calling the read_recipe method for each ID and converting the result to a dictionary.
+    # The resulting list of dictionaries is then returned.
+    return [Recipe.read(rec_id).to_dict() for rec_id in rec_ids]
+
+
+@validate_call
+def api_update_recipe(rec_id: str, **kwargs) -> bool:
+    """
+    Updates a recipe with the given keyword arguments.
+
+    This function takes a recipe ID and arbitrary keyword arguments, checks if the recipe exists,
+    and updates the fields of the recipe with the provided values. If the recipe does not exist,
+    a RuntimeError is raised. If the update is successful, it returns True.
+
+    Args:
+        rec_id (str): The ID of the recipe to update.
+        **kwargs: Arbitrary keyword arguments representing the fields to update.
+
+    Returns:
+        bool: True if the recipe was successfully updated.
+
+    Raises:
+        RuntimeError: If the recipe with the given ID does not exist.
+    """
+    # Check if the recipe exists
+    try:
+        existing_recipe = Recipe.read(rec_id)
+    except Exception:
+        raise RuntimeError(f"Recipe with ID '{rec_id}' does not exist")
+
+    # Update the fields of the existing recipe with the provided kwargs
+    for key, value in kwargs.items():
+        if hasattr(existing_recipe, key):
+            setattr(existing_recipe, key, value)
+
+    # Perform pydantic check on the updated existing recipe
+    RecipeArguments.model_validate(existing_recipe.to_dict())
+
+    # Update the recipe
+    return Recipe.update(existing_recipe)
+
+
+@validate_call
+def api_delete_recipe(rec_id: str) -> bool:
+    """
+    Deletes a recipe identified by its unique recipe ID.
+
+    This function takes a recipe ID, verifies the existence of the recipe, and if found, calls the delete method from
+    the Recipe class to remove the recipe from storage.
+
+    If the deletion is successful, it returns True.
+    If the recipe does not exist or an exception occurs during deletion, a RuntimeError is raised with an
+    appropriate error message.
+
+    Args:
+        rec_id (str): The unique identifier for the recipe to be deleted.
+
+    Returns:
+        bool: True if the recipe was successfully deleted.
+
+    Raises:
+        RuntimeError: If the deletion process encounters an error.
+    """
+    return Recipe.delete(rec_id)
+
+
+def api_get_all_recipe() -> list[dict]:
+    """
+    Retrieves all available recipes.
+
+    This function calls the get_available_recipes method to retrieve all available recipes. It then converts each
+    recipe into a dictionary using the to_dict method and returns a list of these dictionaries.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing a recipe.
+    """
+    _, recipes = Recipe.get_available_items()
+    return [recipe.to_dict() for recipe in recipes]
+
+
+def api_get_all_recipe_name() -> list[str]:
+    """
+    Retrieves all available recipe names.
+
+    This function calls the get_available_recipes method to retrieve all available recipes. It then extracts the names
+    of each recipe and returns a list of these names.
+
+    Returns:
+        list[str]: A list of strings, each representing a recipe name.
+    """
+    recipes_names, _ = Recipe.get_available_items()
+    return recipes_names

moonshot/src/api/api_red_teaming.py

@@ -0,0 +1,59 @@
+from pydantic import validate_call
+
+from moonshot.src.redteaming.attack.attack_module import AttackModule
+
+
+# ------------------------------------------------------------------------------
+# Red teaming APIs
+# ------------------------------------------------------------------------------
+def api_get_all_attack_modules() -> list[str]:
+    """
+    Retrieves all available attack module IDs.
+
+    This function calls the `get_available_items` method from the `AttackModule` class to retrieve all available
+    attack modules.
+
+    It then extracts the IDs of each attack module and returns a list of these IDs.
+
+    Returns:
+        list[str]: A list of strings, each representing an attack module ID.
+    """
+    attack_modules_ids, _ = AttackModule.get_available_items()
+    return attack_modules_ids
+
+
+def api_get_all_attack_module_metadata() -> list[dict]:
+    """
+    Retrieves metadata for all available attack modules.
+
+    This function calls the `get_available_items` method from the `AttackModule` class to retrieve all available
+    attack modules metadata.
+
+    It then extracts the metadata for each attack module and returns a list of dictionaries, each containing the
+    metadata of an attack module.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing the metadata of an attack module.
+    """
+    _, attack_modules_metadata = AttackModule.get_available_items()
+    return attack_modules_metadata
+
+
+@validate_call
+def api_delete_attack_module(am_id: str) -> bool:
+    """
+    Deletes an attack module by its identifier.
+
+    This function takes an attack module ID as input and calls the delete method from the AttackModule class
+    to remove the specified attack module from storage.
+
+    Args:
+        am_id (str): The unique identifier of the attack module to be deleted.
+
+    Returns:
+        bool: True if the attack module was successfully deleted.
+
+    Raises:
+        Exception: If the deletion process encounters an error.
+    """
+    return AttackModule.delete(am_id)

moonshot/src/api/api_result.py

@@ -0,0 +1,84 @@
+from pydantic import conlist, validate_call
+
+from moonshot.src.results.result import Result
+
+
+# ------------------------------------------------------------------------------
+# Result APIs
+# ------------------------------------------------------------------------------
+@validate_call
+def api_read_result(res_id: str) -> dict:
+    """
+    Reads a result and returns its information.
+
+    This function takes a result ID as input, reads the corresponding database file from the storage manager,
+    and returns a dictionary containing the result's information.
+
+    Args:
+        res_id (str): The ID of the result.
+
+    Returns:
+        dict: A dictionary containing the result's information.
+    """
+    return Result.read(res_id)
+
+
+@validate_call
+def api_read_results(res_ids: conlist(str, min_length=1)) -> list[dict]:
+    """
+    Reads multiple results and returns their information.
+
+    This function takes a list of result IDs as input, reads the corresponding database files from the storage manager,
+    and returns a list of dictionaries, each containing a result's information.
+
+    Args:
+        res_ids (conlist(str, min_length=1)): A list of result IDs.
+
+    Returns:
+        list[dict]: A list of dictionaries, each containing a result's information.
+    """
+
+    return [Result.read(res_id) for res_id in res_ids]
+
+
+@validate_call
+def api_delete_result(res_id: str) -> bool:
+    """
+    Deletes a result by its identifier.
+
+    This function takes a result ID as input and calls the delete method from the Result class
+    to remove the specified result from storage.
+
+    Args:
+        res_id (str): The unique identifier of the result to be deleted.
+
+    Returns:
+        bool: True if the result was successfully deleted.
+
+    Raises:
+        Exception: If the deletion process encounters an error.
+    """
+    return Result.delete(res_id)
+
+
+def api_get_all_result() -> list[dict]:
+    """
+    This function retrieves all available results and returns them as a list of dictionaries. Each dictionary
+    represents a result and contains its information.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing a result.
+    """
+    _, results = Result.get_available_items()
+    return [result for result in results]
+
+
+def api_get_all_result_name() -> list[str]:
+    """
+    This function retrieves all available result names and returns them as a list.
+
+    Returns:
+        list[str]: A list of result names.
+    """
+    results_name, _ = Result.get_available_items()
+    return results_name

moonshot/src/api/api_run.py

@@ -0,0 +1,74 @@
+from moonshot.src.api.api_runner import api_get_all_runner, api_load_runner
+from moonshot.src.runs.run import Run
+from moonshot.src.runs.run_arguments import RunArguments
+
+
+# ------------------------------------------------------------------------------
+# Run APIs
+# ------------------------------------------------------------------------------
+def api_get_all_run(runner_id: str = "") -> list[dict]:
+    """
+    Retrieves all runs for a given runner ID or for all runners if no ID is provided.
+
+    This function calls an internal API to get available runs and then converts each run into a dictionary format.
+
+    Args:
+        runner_id (str, optional): The ID of the runner to retrieve runs for. If empty, runs for all runners
+        are retrieved.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing a run's data.
+    """
+    _, runs = _api_get_available_runs(runner_id)
+    return [run.to_dict() for run in runs]
+
+
+def _api_get_available_runs(
+    runner_id: str = "",
+) -> tuple[list[str], list[RunArguments]]:
+    """
+    Retrieves available runs and their corresponding runner IDs.
+
+    This function fetches information about available runs based on the provided runner ID. If no runner ID is
+    provided, it fetches runs for all runners. It returns a tuple containing a list of runner IDs and a list of
+    RunArguments instances representing the runs.
+
+    Args:
+        runner_id (str): The ID of the runner for which to retrieve run information. If empty, information for
+                         all runners is retrieved.
+
+    Returns:
+        tuple[list[str], list[RunArguments]]: A tuple containing a list of runner IDs and a list of RunArguments
+                                              instances for the corresponding runs.
+    """
+    # Lists to hold runner IDs and Run instances.
+    runners_ids = []
+    runs = []
+
+    # Retrieve information for all runners.
+    runners_info = api_get_all_runner()
+
+    # Load the specified runner, or all runners if no ID is provided.
+    if runner_id:
+        runner_instances = (
+            [api_load_runner(runner_id)]
+            if any(runner_id == runner_info.get("id") for runner_info in runners_info)
+            else []
+        )
+    else:
+        runner_instances = [
+            api_load_runner(str(runner_info.get("id"))) for runner_info in runners_info
+        ]
+
+    # Collect runs from each runner instance.
+    for runner_instance in runner_instances:
+        # Only proceed if the runner has an associated database instance.
+        if runner_instance.database_instance:
+            # Fetch all runs from the database associated with the runner.
+            all_runs = Run.get_all_runs(runner_instance.database_instance)
+            # Record the runner ID and the runs.
+            runners_ids.append(runner_instance.id)
+            runs.extend(all_runs)
+
+    # Return the runner IDs and their runs.
+    return runners_ids, runs

moonshot/src/api/api_runner.py

@@ -0,0 +1,132 @@
+from typing import Callable
+
+from pydantic import validate_call
+
+from moonshot.src.runners.runner import Runner
+from moonshot.src.runners.runner_arguments import RunnerArguments
+
+
+# ------------------------------------------------------------------------------
+# Runner APIs
+# ------------------------------------------------------------------------------
+@validate_call
+def api_create_runner(
+    name: str,
+    endpoints: list[str],
+    description: str = "",
+    progress_callback_func: Callable | None = None,
+) -> Runner:
+    """
+    Creates a new runner.
+
+    This function takes the name, endpoints, and an optional progress callback function to create a new Runner instance.
+    The id of the runner is generated from the name of the runner using the slugify function,
+    so it does not need to be provided.
+
+    Args:
+        name (str): The name of the runner.
+        endpoints (list[str]): A list of endpoint identifiers for the runner.
+        description (str, optional): A brief description of the runner. Defaults to an empty string.
+        progress_callback_func (Callable | None, optional): An optional callback function for progress updates.
+        Defaults to None.
+
+    Returns:
+        Runner: A new Runner object.
+    """
+    # Create a new recipe runner
+    # We do not need to provide the id.
+    # This is because during creating:
+    # 1. the id is slugify from the name and stored as id.
+    runner_args = RunnerArguments(
+        id="",
+        name=name,
+        endpoints=endpoints,
+        description=description,
+        progress_callback_func=progress_callback_func,
+    )
+    return Runner.create(runner_args)
+
+
+@validate_call
+def api_load_runner(
+    runner_id: str, progress_callback_func: Callable | None = None
+) -> Runner:
+    """
+    Loads a runner based on the provided runner ID.
+
+    This function retrieves the runner using the provided runner ID and then loads it.
+    It utilizes the Runner's load method to fetch and return the runner.
+
+    Args:
+        runner_id (str): The ID of the runner to be loaded.
+        progress_callback_func (Callable | None): The progress callback function to be used by the runner.
+
+    Returns:
+        Runner: An initialized Runner object.
+    """
+    return Runner.load(runner_id, progress_callback_func)
+
+
+@validate_call
+def api_read_runner(runner_id: str) -> dict:
+    """
+    Reads a runner and returns its information.
+
+    This function takes a runner ID as input, reads the corresponding runner,
+    and returns a dictionary containing the runner's information.
+
+    Args:
+        runner_id (str): The ID of the runner.
+
+    Returns:
+        dict: A dictionary containing the runner's information.
+    """
+    return Runner.read(runner_id).to_dict()
+
+
+@validate_call
+def api_delete_runner(runner_id: str) -> bool:
+    """
+    Deletes a runner by its identifier.
+
+    This function takes a runner ID as input and calls the delete method from the Runner class
+    to remove the specified runner from storage.
+
+    Args:
+        runner_id (str): The unique identifier of the runner to be deleted.
+
+    Returns:
+        bool: True if the runner was successfully deleted.
+
+    Raises:
+        Exception: If the deletion process encounters an error.
+    """
+    return Runner.delete(runner_id)
+
+
+def api_get_all_runner() -> list[dict]:
+    """
+    Retrieves all available runners.
+
+    This function calls the get_available_items method to retrieve all available runners. It then converts each
+    runner into a dictionary using the to_dict method and returns a list of these dictionaries.
+
+    Returns:
+        list[dict]: A list of dictionaries, each representing a runner.
+    """
+    _, runners = Runner.get_available_items()
+    return [runner.to_dict() for runner in runners]
+
+
+def api_get_all_runner_name() -> list[str]:
+    """
+    Retrieves all available runner names.
+
+    This function calls the get_available_items method to retrieve all available runners. It then extracts the names of
+    each runner and returns a list of these names.
+
+    Returns:
+        list[str]: A list of runner names.
+    """
+    runners_names, _ = Runner.get_available_items()
+    return runners_names