aiverify-moonshot 0.4.1__py3-none-any.whl → 0.4.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. {aiverify_moonshot-0.4.1.dist-info → aiverify_moonshot-0.4.3.dist-info}/METADATA +2 -2
  2. {aiverify_moonshot-0.4.1.dist-info → aiverify_moonshot-0.4.3.dist-info}/RECORD +70 -56
  3. moonshot/__main__.py +77 -35
  4. moonshot/api.py +16 -0
  5. moonshot/integrations/cli/benchmark/benchmark.py +29 -13
  6. moonshot/integrations/cli/benchmark/cookbook.py +62 -24
  7. moonshot/integrations/cli/benchmark/datasets.py +79 -40
  8. moonshot/integrations/cli/benchmark/metrics.py +62 -23
  9. moonshot/integrations/cli/benchmark/recipe.py +89 -69
  10. moonshot/integrations/cli/benchmark/result.py +85 -47
  11. moonshot/integrations/cli/benchmark/run.py +99 -59
  12. moonshot/integrations/cli/common/common.py +20 -6
  13. moonshot/integrations/cli/common/connectors.py +154 -74
  14. moonshot/integrations/cli/common/dataset.py +66 -0
  15. moonshot/integrations/cli/common/prompt_template.py +57 -19
  16. moonshot/integrations/cli/redteam/attack_module.py +90 -24
  17. moonshot/integrations/cli/redteam/context_strategy.py +83 -23
  18. moonshot/integrations/cli/redteam/prompt_template.py +1 -1
  19. moonshot/integrations/cli/redteam/redteam.py +52 -6
  20. moonshot/integrations/cli/redteam/session.py +565 -44
  21. moonshot/integrations/cli/utils/process_data.py +52 -0
  22. moonshot/integrations/web_api/__main__.py +2 -0
  23. moonshot/integrations/web_api/app.py +6 -6
  24. moonshot/integrations/web_api/container.py +12 -2
  25. moonshot/integrations/web_api/routes/bookmark.py +173 -0
  26. moonshot/integrations/web_api/routes/dataset.py +46 -1
  27. moonshot/integrations/web_api/schemas/bookmark_create_dto.py +13 -0
  28. moonshot/integrations/web_api/schemas/dataset_create_dto.py +18 -0
  29. moonshot/integrations/web_api/schemas/recipe_create_dto.py +0 -2
  30. moonshot/integrations/web_api/services/bookmark_service.py +94 -0
  31. moonshot/integrations/web_api/services/dataset_service.py +25 -0
  32. moonshot/integrations/web_api/services/recipe_service.py +0 -1
  33. moonshot/integrations/web_api/services/utils/file_manager.py +52 -0
  34. moonshot/integrations/web_api/status_updater/moonshot_ui_webhook.py +0 -1
  35. moonshot/integrations/web_api/temp/.gitkeep +0 -0
  36. moonshot/src/api/api_bookmark.py +95 -0
  37. moonshot/src/api/api_connector_endpoint.py +1 -1
  38. moonshot/src/api/api_context_strategy.py +2 -2
  39. moonshot/src/api/api_dataset.py +35 -0
  40. moonshot/src/api/api_recipe.py +0 -3
  41. moonshot/src/api/api_session.py +1 -1
  42. moonshot/src/bookmark/bookmark.py +257 -0
  43. moonshot/src/bookmark/bookmark_arguments.py +38 -0
  44. moonshot/src/configs/env_variables.py +12 -2
  45. moonshot/src/connectors/connector.py +15 -7
  46. moonshot/src/connectors_endpoints/connector_endpoint.py +65 -49
  47. moonshot/src/cookbooks/cookbook.py +57 -37
  48. moonshot/src/datasets/dataset.py +125 -5
  49. moonshot/src/metrics/metric.py +8 -4
  50. moonshot/src/metrics/metric_interface.py +8 -2
  51. moonshot/src/prompt_templates/prompt_template.py +5 -1
  52. moonshot/src/recipes/recipe.py +38 -40
  53. moonshot/src/recipes/recipe_arguments.py +0 -4
  54. moonshot/src/redteaming/attack/attack_module.py +18 -8
  55. moonshot/src/redteaming/attack/context_strategy.py +6 -2
  56. moonshot/src/redteaming/session/session.py +15 -11
  57. moonshot/src/results/result.py +7 -3
  58. moonshot/src/runners/runner.py +65 -42
  59. moonshot/src/runs/run.py +15 -11
  60. moonshot/src/runs/run_progress.py +7 -3
  61. moonshot/src/storage/db_interface.py +14 -0
  62. moonshot/src/storage/storage.py +33 -2
  63. moonshot/src/utils/find_feature.py +45 -0
  64. moonshot/src/utils/log.py +72 -0
  65. moonshot/src/utils/pagination.py +25 -0
  66. moonshot/src/utils/timeit.py +8 -1
  67. {aiverify_moonshot-0.4.1.dist-info → aiverify_moonshot-0.4.3.dist-info}/WHEEL +0 -0
  68. {aiverify_moonshot-0.4.1.dist-info → aiverify_moonshot-0.4.3.dist-info}/licenses/AUTHORS.md +0 -0
  69. {aiverify_moonshot-0.4.1.dist-info → aiverify_moonshot-0.4.3.dist-info}/licenses/LICENSE.md +0 -0
  70. {aiverify_moonshot-0.4.1.dist-info → aiverify_moonshot-0.4.3.dist-info}/licenses/NOTICES.md +0 -0
@@ -1,40 +1,49 @@
1
+ from ast import literal_eval
2
+
1
3
  import cmd2
2
4
  from rich.console import Console
3
5
  from rich.table import Table
4
6
 
5
7
  from moonshot.api import api_delete_attack_module, api_get_all_attack_module_metadata
8
+ from moonshot.integrations.cli.utils.process_data import filter_data
6
9
 
7
10
  console = Console()
8
11
 
9
12
 
10
- def list_attack_modules() -> None:
13
+ # ------------------------------------------------------------------------------
14
+ # CLI Functions
15
+ # ------------------------------------------------------------------------------
16
+ def list_attack_modules(args) -> list | None:
11
17
  """
12
18
  Retrieves and prints the metadata of all attack modules.
19
+
20
+ Args:
21
+ args: A namespace object from argparse. It should have an optional attribute:
22
+ find (str): Optional field to find attack module(s) with a keyword.
23
+ pagination (str): Optional field to paginate attack modules.
24
+
25
+ Returns:
26
+ list | None: A list of AttackModule or None if there is no result.
13
27
  """
14
- print("Listing attack modules may take a while...")
15
- attack_module_metadata_list = api_get_all_attack_module_metadata()
16
-
17
- if attack_module_metadata_list:
18
- table = Table(
19
- title="Attack Module List",
20
- show_lines=True,
21
- expand=True,
22
- header_style="bold",
23
- )
24
- table.add_column("No.", width=2)
25
- table.add_column("Details", justify="left", width=98)
26
-
27
- for attack_module_index, attack_module_data in enumerate(
28
- attack_module_metadata_list, 1
29
- ):
30
- attack_module_data_str = ""
31
- for k, v in attack_module_data.items():
32
- attack_module_data_str += f"[blue]{k.capitalize()}:[/blue] {v}\n\n"
33
- table.add_row(str(attack_module_index), attack_module_data_str)
28
+ try:
29
+ print("Listing attack modules may take a while...")
30
+ attack_module_metadata_list = api_get_all_attack_module_metadata()
31
+ keyword = args.find.lower() if args.find else ""
32
+ pagination = literal_eval(args.pagination) if args.pagination else ()
34
33
 
35
- console.print(table)
36
- else:
37
- console.print("[red]There are no attack modules found.[/red]", style="bold")
34
+ if attack_module_metadata_list:
35
+ filtered_attack_modules_list = filter_data(
36
+ attack_module_metadata_list, keyword, pagination
37
+ )
38
+ if filtered_attack_modules_list:
39
+ _display_attack_modules(filtered_attack_modules_list)
40
+ return filtered_attack_modules_list
41
+
42
+ console.print("[red]There are no attack modules found.[/red]")
43
+ return None
44
+
45
+ except Exception as e:
46
+ print(f"[list_attack_modules]: {str(e)}")
38
47
 
39
48
 
40
49
  def delete_attack_module(args) -> None:
@@ -59,6 +68,41 @@ def delete_attack_module(args) -> None:
59
68
  print(f"[delete_attack_module]: {str(e)}")
60
69
 
61
70
 
71
+ # ------------------------------------------------------------------------------
72
+ # Helper functions: Display on cli
73
+ # ------------------------------------------------------------------------------
74
+ def _display_attack_modules(attack_modules: list) -> None:
75
+ """
76
+ Display a list of attack modules.
77
+
78
+ This function takes a list of attack modules and displays them in a table format. If the list is empty, it prints a
79
+ message indicating that no attack modules were found.
80
+
81
+ Args:
82
+ attack_modules (list): A list of attack modules.
83
+
84
+ Returns:
85
+ None
86
+ """
87
+ table = Table(
88
+ title="Attack Module List",
89
+ show_lines=True,
90
+ expand=True,
91
+ header_style="bold",
92
+ )
93
+ table.add_column("No.", width=2)
94
+ table.add_column("Details", justify="left", width=98)
95
+
96
+ for idx, attack_module_data in enumerate(attack_modules, 1):
97
+ attack_module_data_str = ""
98
+ for k, v in attack_module_data.items():
99
+ if k != "idx":
100
+ attack_module_data_str += f"[blue]{k.capitalize()}:[/blue] {v}\n\n"
101
+ idx = attack_module_data.get("idx", idx)
102
+ table.add_row(str(idx), attack_module_data_str)
103
+ console.print(table)
104
+
105
+
62
106
  # Delete attack module arguments
63
107
  delete_attack_module_args = cmd2.Cmd2ArgumentParser(
64
108
  description="Delete an attack module.",
@@ -68,3 +112,25 @@ delete_attack_module_args = cmd2.Cmd2ArgumentParser(
68
112
  delete_attack_module_args.add_argument(
69
113
  "attack_module", type=str, help="The ID of the attack module to delete"
70
114
  )
115
+
116
+ # List attack modules arguments
117
+ list_attack_modules_args = cmd2.Cmd2ArgumentParser(
118
+ description="List all attack modules.",
119
+ epilog='Example:\n list_attack_modules -f "text"',
120
+ )
121
+
122
+ list_attack_modules_args.add_argument(
123
+ "-f",
124
+ "--find",
125
+ type=str,
126
+ help="Optional field to find attack module(s) with keyword",
127
+ nargs="?",
128
+ )
129
+
130
+ list_attack_modules_args.add_argument(
131
+ "-p",
132
+ "--pagination",
133
+ type=str,
134
+ help="Optional tuple to paginate attack module(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
135
+ nargs="?",
136
+ )
@@ -1,4 +1,5 @@
1
1
  import argparse
2
+ from ast import literal_eval
2
3
 
3
4
  import cmd2
4
5
  from rich.console import Console
@@ -10,6 +11,7 @@ from moonshot.api import (
10
11
  api_update_context_strategy,
11
12
  )
12
13
  from moonshot.integrations.cli.active_session_cfg import active_session
14
+ from moonshot.integrations.cli.utils.process_data import filter_data
13
15
  from moonshot.src.redteaming.session.session import Session
14
16
 
15
17
  console = Console()
@@ -32,12 +34,12 @@ def use_context_strategy(args: argparse.Namespace) -> None:
32
34
 
33
35
  # Check if current session exists. If it does, update context strategy and number of previous prompts
34
36
  if active_session:
35
- active_session["context_strategy"] = new_context_strategy_name
36
- active_session["cs_num_of_prev_prompts"] = num_of_prev_prompts
37
37
  try:
38
38
  api_update_context_strategy(
39
39
  active_session["session_id"], new_context_strategy_name
40
40
  )
41
+ active_session["context_strategy"] = new_context_strategy_name
42
+ active_session["cs_num_of_prev_prompts"] = num_of_prev_prompts
41
43
  print(
42
44
  f"Updated session: {active_session['session_id']}. "
43
45
  f"Context Strategy: {active_session['context_strategy']}."
@@ -52,30 +54,36 @@ def use_context_strategy(args: argparse.Namespace) -> None:
52
54
  )
53
55
 
54
56
 
55
- def list_context_strategies() -> None:
57
+ def list_context_strategies(args) -> list | None:
56
58
  """
57
59
  List all context strategies available.
60
+
61
+ Args:
62
+ args: A namespace object from argparse. It should have an optional attribute:
63
+ find (str): Optional field to find context strategies with a keyword.
64
+ pagination (str): Optional field to paginate context strategies.
65
+
66
+ Returns:
67
+ list | None: A list of ContextStrategy or None if there is no result.
58
68
  """
59
- context_strategy_metadata_list = api_get_all_context_strategy_metadata()
60
- if context_strategy_metadata_list:
61
- table = Table(
62
- title="Context Strategy List",
63
- show_lines=True,
64
- expand=True,
65
- header_style="bold",
66
- )
67
- table.add_column("No.", justify="left", width=2)
68
- table.add_column("Context Strategy Information", justify="left", width=98)
69
- for context_strategy_index, context_strategy_data in enumerate(
70
- context_strategy_metadata_list, 1
71
- ):
72
- context_strategy_data_str = ""
73
- for k, v in context_strategy_data.items():
74
- context_strategy_data_str += f"[blue]{k.capitalize()}:[/blue] {v}\n\n"
75
- table.add_row(str(context_strategy_index), context_strategy_data_str)
76
- console.print(table)
77
- else:
78
- console.print("[red]There are no context strategies found.[/red]", style="bold")
69
+ try:
70
+ context_strategy_metadata_list = api_get_all_context_strategy_metadata()
71
+ keyword = args.find.lower() if args.find else ""
72
+ pagination = literal_eval(args.pagination) if args.pagination else ()
73
+
74
+ if context_strategy_metadata_list:
75
+ filtered_context_strategies_list = filter_data(
76
+ context_strategy_metadata_list, keyword, pagination
77
+ )
78
+ if filtered_context_strategies_list:
79
+ _display_context_strategies(filtered_context_strategies_list)
80
+ return filtered_context_strategies_list
81
+
82
+ console.print("[red]There are no context strategies found.[/red]")
83
+ return None
84
+
85
+ except Exception as e:
86
+ print(f"[list_context_strategies]: {str(e)}")
79
87
 
80
88
 
81
89
  def clear_context_strategy() -> None:
@@ -118,6 +126,36 @@ def delete_context_strategy(args) -> None:
118
126
  print(f"[delete_context_strategy]: {str(e)}")
119
127
 
120
128
 
129
+ def _display_context_strategies(context_strategies: list) -> None:
130
+ """
131
+ Display a list of context strategies.
132
+
133
+ This function takes a list of context strategies and displays them in a table format. If the list is empty,
134
+ it prints a message indicating that no attack modules were found.
135
+
136
+ Args:
137
+ context_strategies (list): A list of context strategies.
138
+
139
+ Returns:
140
+ None
141
+ """
142
+ table = Table(
143
+ title="Context Strategy List",
144
+ show_lines=True,
145
+ expand=True,
146
+ header_style="bold",
147
+ )
148
+ table.add_column("No.", justify="left", width=2)
149
+ table.add_column("Context Strategy Information", justify="left", width=98)
150
+ for idx, context_strategy_data in enumerate(context_strategies, 1):
151
+ context_strategy_data_str = ""
152
+ for k, v in context_strategy_data.items():
153
+ if k != "idx":
154
+ context_strategy_data_str += f"[blue]{k.capitalize()}:[/blue] {v}\n\n"
155
+ table.add_row(str(idx), context_strategy_data_str)
156
+ console.print(table)
157
+
158
+
121
159
  # Use context strategy arguments
122
160
  use_context_strategy_args = cmd2.Cmd2ArgumentParser(
123
161
  description="Use a context strategy.",
@@ -145,3 +183,25 @@ delete_context_strategy_args = cmd2.Cmd2ArgumentParser(
145
183
  delete_context_strategy_args.add_argument(
146
184
  "context_strategy", type=str, help="The ID of the context strategy to delete"
147
185
  )
186
+
187
+ # List context strategies arguments
188
+ list_context_strategies_args = cmd2.Cmd2ArgumentParser(
189
+ description="List all context strategies.",
190
+ epilog='Example:\n list_context_strategies -f "previous_prompt"',
191
+ )
192
+
193
+ list_context_strategies_args.add_argument(
194
+ "-f",
195
+ "--find",
196
+ type=str,
197
+ help="Optional field to find context strategies with keyword",
198
+ nargs="?",
199
+ )
200
+
201
+ list_context_strategies_args.add_argument(
202
+ "-p",
203
+ "--pagination",
204
+ type=str,
205
+ help="Optional tuple to paginate context strategies(s). E.g. (2,10) returns 2nd page with 10 items in each page.",
206
+ nargs="?",
207
+ )
@@ -20,11 +20,11 @@ def use_prompt_template(args: argparse.Namespace) -> None:
20
20
 
21
21
  # Check if current session exists
22
22
  if active_session:
23
- active_session["prompt_template"] = new_prompt_template_name
24
23
  try:
25
24
  api_update_prompt_template(
26
25
  active_session["session_id"], new_prompt_template_name
27
26
  )
27
+ active_session["prompt_template"] = new_prompt_template_name
28
28
  print(
29
29
  f"Updated session: {active_session['session_id']}. "
30
30
  f"Prompt Template: {active_session['prompt_template']}."
@@ -6,12 +6,14 @@ from moonshot.integrations.cli.redteam.attack_module import (
6
6
  delete_attack_module,
7
7
  delete_attack_module_args,
8
8
  list_attack_modules,
9
+ list_attack_modules_args,
9
10
  )
10
11
  from moonshot.integrations.cli.redteam.context_strategy import (
11
12
  clear_context_strategy,
12
13
  delete_context_strategy,
13
14
  delete_context_strategy_args,
14
15
  list_context_strategies,
16
+ list_context_strategies_args,
15
17
  use_context_strategy,
16
18
  use_context_strategy_args,
17
19
  )
@@ -21,16 +23,30 @@ from moonshot.integrations.cli.redteam.prompt_template import (
21
23
  use_prompt_template_args,
22
24
  )
23
25
  from moonshot.integrations.cli.redteam.session import (
26
+ add_bookmark,
27
+ add_bookmark_args,
24
28
  automated_rt_session_args,
29
+ delete_bookmark,
30
+ delete_bookmark_args,
25
31
  delete_session,
26
32
  delete_session_args,
27
33
  end_session,
34
+ export_bookmarks,
35
+ export_bookmarks_args,
36
+ list_bookmarks,
37
+ list_bookmarks_args,
28
38
  list_sessions,
39
+ list_sessions_args,
29
40
  new_session,
30
41
  new_session_args,
31
42
  run_attack_module,
43
+ show_prompts,
44
+ use_bookmark,
45
+ use_bookmark_args,
32
46
  use_session,
33
47
  use_session_args,
48
+ view_bookmark,
49
+ view_bookmark_args,
34
50
  )
35
51
 
36
52
 
@@ -46,8 +62,9 @@ class RedTeamCommandSet(cmd2.CommandSet):
46
62
  def do_end_session(self, _: cmd2.Statement) -> None:
47
63
  end_session()
48
64
 
49
- def do_list_sessions(self, _: cmd2.Statement) -> None:
50
- list_sessions()
65
+ @cmd2.with_argparser(list_sessions_args)
66
+ def do_list_sessions(self, args: argparse.Namespace) -> None:
67
+ list_sessions(args)
51
68
 
52
69
  @cmd2.with_argparser(use_session_args)
53
70
  def do_use_session(self, args: argparse.Namespace) -> None:
@@ -60,8 +77,9 @@ class RedTeamCommandSet(cmd2.CommandSet):
60
77
  def do_clear_prompt_template(self, _: cmd2.Statement) -> None:
61
78
  clear_prompt_template()
62
79
 
63
- def do_list_context_strategies(self, _: cmd2.Statement) -> None:
64
- list_context_strategies()
80
+ @cmd2.with_argparser(list_context_strategies_args)
81
+ def do_list_context_strategies(self, args: argparse.Namespace) -> None:
82
+ list_context_strategies(args)
65
83
 
66
84
  @cmd2.with_argparser(use_context_strategy_args)
67
85
  def do_use_context_strategy(self, args: argparse.Namespace) -> None:
@@ -74,8 +92,9 @@ class RedTeamCommandSet(cmd2.CommandSet):
74
92
  def do_run_attack_module(self, args: argparse.Namespace) -> None:
75
93
  run_attack_module(args)
76
94
 
77
- def do_list_attack_modules(self, _: cmd2.Statement) -> None:
78
- list_attack_modules()
95
+ @cmd2.with_argparser(list_attack_modules_args)
96
+ def do_list_attack_modules(self, args: argparse.Namespace) -> None:
97
+ list_attack_modules(args)
79
98
 
80
99
  @cmd2.with_argparser(delete_session_args)
81
100
  def do_delete_session(self, args: argparse.Namespace) -> None:
@@ -88,3 +107,30 @@ class RedTeamCommandSet(cmd2.CommandSet):
88
107
  @cmd2.with_argparser(delete_attack_module_args)
89
108
  def do_delete_attack_module(self, args: argparse.Namespace) -> None:
90
109
  delete_attack_module(args)
110
+
111
+ @cmd2.with_argparser(add_bookmark_args)
112
+ def do_add_bookmark(self, args: argparse.Namespace) -> None:
113
+ add_bookmark(args)
114
+
115
+ @cmd2.with_argparser(use_bookmark_args)
116
+ def do_use_bookmark(self, args: argparse.Namespace) -> None:
117
+ use_bookmark(args)
118
+
119
+ @cmd2.with_argparser(delete_bookmark_args)
120
+ def do_delete_bookmark(self, args: argparse.Namespace) -> None:
121
+ delete_bookmark(args)
122
+
123
+ @cmd2.with_argparser(list_bookmarks_args)
124
+ def do_list_bookmarks(self, args: argparse.Namespace) -> None:
125
+ list_bookmarks(args)
126
+
127
+ @cmd2.with_argparser(view_bookmark_args)
128
+ def do_view_bookmark(self, args: argparse.Namespace) -> None:
129
+ view_bookmark(args)
130
+
131
+ @cmd2.with_argparser(export_bookmarks_args)
132
+ def do_export_bookmarks(self, args: argparse.Namespace) -> None:
133
+ export_bookmarks(args)
134
+
135
+ def do_show_prompts(self, _: cmd2.Statement) -> None:
136
+ show_prompts()