aiverify-moonshot 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- aiverify_moonshot-0.4.0.dist-info/METADATA +249 -0
- aiverify_moonshot-0.4.0.dist-info/RECORD +163 -0
- aiverify_moonshot-0.4.0.dist-info/WHEEL +4 -0
- aiverify_moonshot-0.4.0.dist-info/licenses/AUTHORS.md +5 -0
- aiverify_moonshot-0.4.0.dist-info/licenses/LICENSE.md +201 -0
- aiverify_moonshot-0.4.0.dist-info/licenses/NOTICES.md +3340 -0
- moonshot/__init__.py +0 -0
- moonshot/__main__.py +198 -0
- moonshot/api.py +155 -0
- moonshot/integrations/__init__.py +0 -0
- moonshot/integrations/cli/__init__.py +0 -0
- moonshot/integrations/cli/__main__.py +25 -0
- moonshot/integrations/cli/active_session_cfg.py +1 -0
- moonshot/integrations/cli/benchmark/__init__.py +0 -0
- moonshot/integrations/cli/benchmark/benchmark.py +186 -0
- moonshot/integrations/cli/benchmark/cookbook.py +545 -0
- moonshot/integrations/cli/benchmark/datasets.py +164 -0
- moonshot/integrations/cli/benchmark/metrics.py +141 -0
- moonshot/integrations/cli/benchmark/recipe.py +598 -0
- moonshot/integrations/cli/benchmark/result.py +216 -0
- moonshot/integrations/cli/benchmark/run.py +140 -0
- moonshot/integrations/cli/benchmark/runner.py +174 -0
- moonshot/integrations/cli/cli.py +64 -0
- moonshot/integrations/cli/common/__init__.py +0 -0
- moonshot/integrations/cli/common/common.py +72 -0
- moonshot/integrations/cli/common/connectors.py +325 -0
- moonshot/integrations/cli/common/display_helper.py +42 -0
- moonshot/integrations/cli/common/prompt_template.py +94 -0
- moonshot/integrations/cli/initialisation/__init__.py +0 -0
- moonshot/integrations/cli/initialisation/initialisation.py +14 -0
- moonshot/integrations/cli/redteam/__init__.py +0 -0
- moonshot/integrations/cli/redteam/attack_module.py +70 -0
- moonshot/integrations/cli/redteam/context_strategy.py +147 -0
- moonshot/integrations/cli/redteam/prompt_template.py +67 -0
- moonshot/integrations/cli/redteam/redteam.py +90 -0
- moonshot/integrations/cli/redteam/session.py +467 -0
- moonshot/integrations/web_api/.env.dev +7 -0
- moonshot/integrations/web_api/__init__.py +0 -0
- moonshot/integrations/web_api/__main__.py +56 -0
- moonshot/integrations/web_api/app.py +125 -0
- moonshot/integrations/web_api/container.py +146 -0
- moonshot/integrations/web_api/log/.gitkeep +0 -0
- moonshot/integrations/web_api/logging_conf.py +114 -0
- moonshot/integrations/web_api/routes/__init__.py +0 -0
- moonshot/integrations/web_api/routes/attack_modules.py +66 -0
- moonshot/integrations/web_api/routes/benchmark.py +116 -0
- moonshot/integrations/web_api/routes/benchmark_result.py +175 -0
- moonshot/integrations/web_api/routes/context_strategy.py +129 -0
- moonshot/integrations/web_api/routes/cookbook.py +225 -0
- moonshot/integrations/web_api/routes/dataset.py +120 -0
- moonshot/integrations/web_api/routes/endpoint.py +282 -0
- moonshot/integrations/web_api/routes/metric.py +78 -0
- moonshot/integrations/web_api/routes/prompt_template.py +128 -0
- moonshot/integrations/web_api/routes/recipe.py +219 -0
- moonshot/integrations/web_api/routes/redteam.py +609 -0
- moonshot/integrations/web_api/routes/runner.py +239 -0
- moonshot/integrations/web_api/schemas/__init__.py +0 -0
- moonshot/integrations/web_api/schemas/benchmark_runner_dto.py +13 -0
- moonshot/integrations/web_api/schemas/cookbook_create_dto.py +19 -0
- moonshot/integrations/web_api/schemas/cookbook_response_model.py +9 -0
- moonshot/integrations/web_api/schemas/dataset_response_dto.py +9 -0
- moonshot/integrations/web_api/schemas/endpoint_create_dto.py +21 -0
- moonshot/integrations/web_api/schemas/endpoint_response_model.py +11 -0
- moonshot/integrations/web_api/schemas/prompt_response_model.py +14 -0
- moonshot/integrations/web_api/schemas/prompt_template_response_model.py +10 -0
- moonshot/integrations/web_api/schemas/recipe_create_dto.py +32 -0
- moonshot/integrations/web_api/schemas/recipe_response_model.py +7 -0
- moonshot/integrations/web_api/schemas/session_create_dto.py +16 -0
- moonshot/integrations/web_api/schemas/session_prompt_dto.py +7 -0
- moonshot/integrations/web_api/schemas/session_response_model.py +38 -0
- moonshot/integrations/web_api/services/__init__.py +0 -0
- moonshot/integrations/web_api/services/attack_module_service.py +34 -0
- moonshot/integrations/web_api/services/auto_red_team_test_manager.py +86 -0
- moonshot/integrations/web_api/services/auto_red_team_test_state.py +57 -0
- moonshot/integrations/web_api/services/base_service.py +8 -0
- moonshot/integrations/web_api/services/benchmark_result_service.py +25 -0
- moonshot/integrations/web_api/services/benchmark_test_manager.py +106 -0
- moonshot/integrations/web_api/services/benchmark_test_state.py +56 -0
- moonshot/integrations/web_api/services/benchmarking_service.py +31 -0
- moonshot/integrations/web_api/services/context_strategy_service.py +22 -0
- moonshot/integrations/web_api/services/cookbook_service.py +194 -0
- moonshot/integrations/web_api/services/dataset_service.py +20 -0
- moonshot/integrations/web_api/services/endpoint_service.py +65 -0
- moonshot/integrations/web_api/services/metric_service.py +14 -0
- moonshot/integrations/web_api/services/prompt_template_service.py +39 -0
- moonshot/integrations/web_api/services/recipe_service.py +155 -0
- moonshot/integrations/web_api/services/runner_service.py +147 -0
- moonshot/integrations/web_api/services/session_service.py +350 -0
- moonshot/integrations/web_api/services/utils/exceptions_handler.py +41 -0
- moonshot/integrations/web_api/services/utils/results_formatter.py +47 -0
- moonshot/integrations/web_api/status_updater/interface/benchmark_progress_callback.py +14 -0
- moonshot/integrations/web_api/status_updater/interface/redteam_progress_callback.py +14 -0
- moonshot/integrations/web_api/status_updater/moonshot_ui_webhook.py +72 -0
- moonshot/integrations/web_api/types/types.py +99 -0
- moonshot/src/__init__.py +0 -0
- moonshot/src/api/__init__.py +0 -0
- moonshot/src/api/api_connector.py +58 -0
- moonshot/src/api/api_connector_endpoint.py +162 -0
- moonshot/src/api/api_context_strategy.py +57 -0
- moonshot/src/api/api_cookbook.py +160 -0
- moonshot/src/api/api_dataset.py +46 -0
- moonshot/src/api/api_environment_variables.py +17 -0
- moonshot/src/api/api_metrics.py +51 -0
- moonshot/src/api/api_prompt_template.py +43 -0
- moonshot/src/api/api_recipe.py +182 -0
- moonshot/src/api/api_red_teaming.py +59 -0
- moonshot/src/api/api_result.py +84 -0
- moonshot/src/api/api_run.py +74 -0
- moonshot/src/api/api_runner.py +132 -0
- moonshot/src/api/api_session.py +290 -0
- moonshot/src/configs/__init__.py +0 -0
- moonshot/src/configs/env_variables.py +187 -0
- moonshot/src/connectors/__init__.py +0 -0
- moonshot/src/connectors/connector.py +327 -0
- moonshot/src/connectors/connector_prompt_arguments.py +17 -0
- moonshot/src/connectors_endpoints/__init__.py +0 -0
- moonshot/src/connectors_endpoints/connector_endpoint.py +211 -0
- moonshot/src/connectors_endpoints/connector_endpoint_arguments.py +54 -0
- moonshot/src/cookbooks/__init__.py +0 -0
- moonshot/src/cookbooks/cookbook.py +225 -0
- moonshot/src/cookbooks/cookbook_arguments.py +34 -0
- moonshot/src/datasets/__init__.py +0 -0
- moonshot/src/datasets/dataset.py +255 -0
- moonshot/src/datasets/dataset_arguments.py +50 -0
- moonshot/src/metrics/__init__.py +0 -0
- moonshot/src/metrics/metric.py +192 -0
- moonshot/src/metrics/metric_interface.py +95 -0
- moonshot/src/prompt_templates/__init__.py +0 -0
- moonshot/src/prompt_templates/prompt_template.py +103 -0
- moonshot/src/recipes/__init__.py +0 -0
- moonshot/src/recipes/recipe.py +340 -0
- moonshot/src/recipes/recipe_arguments.py +111 -0
- moonshot/src/redteaming/__init__.py +0 -0
- moonshot/src/redteaming/attack/__init__.py +0 -0
- moonshot/src/redteaming/attack/attack_module.py +618 -0
- moonshot/src/redteaming/attack/attack_module_arguments.py +44 -0
- moonshot/src/redteaming/attack/context_strategy.py +131 -0
- moonshot/src/redteaming/context_strategy/__init__.py +0 -0
- moonshot/src/redteaming/context_strategy/context_strategy_interface.py +46 -0
- moonshot/src/redteaming/session/__init__.py +0 -0
- moonshot/src/redteaming/session/chat.py +209 -0
- moonshot/src/redteaming/session/red_teaming_progress.py +128 -0
- moonshot/src/redteaming/session/red_teaming_type.py +6 -0
- moonshot/src/redteaming/session/session.py +775 -0
- moonshot/src/results/__init__.py +0 -0
- moonshot/src/results/result.py +119 -0
- moonshot/src/results/result_arguments.py +44 -0
- moonshot/src/runners/__init__.py +0 -0
- moonshot/src/runners/runner.py +476 -0
- moonshot/src/runners/runner_arguments.py +46 -0
- moonshot/src/runners/runner_type.py +6 -0
- moonshot/src/runs/__init__.py +0 -0
- moonshot/src/runs/run.py +344 -0
- moonshot/src/runs/run_arguments.py +162 -0
- moonshot/src/runs/run_progress.py +145 -0
- moonshot/src/runs/run_status.py +10 -0
- moonshot/src/storage/__init__.py +0 -0
- moonshot/src/storage/db_interface.py +128 -0
- moonshot/src/storage/io_interface.py +31 -0
- moonshot/src/storage/storage.py +525 -0
- moonshot/src/utils/__init__.py +0 -0
- moonshot/src/utils/import_modules.py +96 -0
- moonshot/src/utils/timeit.py +25 -0
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
import importlib.util
|
|
2
|
+
import inspect
|
|
3
|
+
from importlib.machinery import ModuleSpec
|
|
4
|
+
from types import ModuleType
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def create_module_spec(module_name: str, module_file_path: str) -> None | ModuleSpec:
|
|
9
|
+
"""
|
|
10
|
+
A helper method to create module specifications if it does not exist
|
|
11
|
+
|
|
12
|
+
Args:
|
|
13
|
+
module_name (str): Input module name to be imported
|
|
14
|
+
module_file_path (str): Input module file path to be imported
|
|
15
|
+
|
|
16
|
+
Returns:
|
|
17
|
+
None | ModuleSpec: Generated module specifications for importing or error
|
|
18
|
+
"""
|
|
19
|
+
try:
|
|
20
|
+
if (
|
|
21
|
+
module_name is None
|
|
22
|
+
or not isinstance(module_name, str)
|
|
23
|
+
or module_file_path is None
|
|
24
|
+
or not isinstance(module_file_path, str)
|
|
25
|
+
):
|
|
26
|
+
return None
|
|
27
|
+
|
|
28
|
+
module_spec = importlib.util.find_spec(module_name)
|
|
29
|
+
if module_spec is None:
|
|
30
|
+
# Create a module spec since it is not available
|
|
31
|
+
module_spec = importlib.util.spec_from_file_location(
|
|
32
|
+
module_name, module_file_path
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
return module_spec
|
|
36
|
+
|
|
37
|
+
except ValueError:
|
|
38
|
+
# Unable to find spec from this file to create
|
|
39
|
+
return None
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def import_module_from_spec(module_spec: ModuleSpec) -> ModuleType | None:
|
|
43
|
+
"""
|
|
44
|
+
A helper method to import python module using module specifications
|
|
45
|
+
|
|
46
|
+
Args:
|
|
47
|
+
module_spec (ModuleSpec): A generated module specifications for the module to be imported
|
|
48
|
+
|
|
49
|
+
Returns:
|
|
50
|
+
ModuleType | None: An imported module
|
|
51
|
+
"""
|
|
52
|
+
if module_spec is None or not isinstance(module_spec, ModuleSpec):
|
|
53
|
+
return None
|
|
54
|
+
|
|
55
|
+
module = importlib.util.module_from_spec(module_spec)
|
|
56
|
+
module_spec.loader.exec_module(module)
|
|
57
|
+
return module
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def get_instance(id: str, filepath: str) -> Any:
|
|
61
|
+
"""
|
|
62
|
+
A helper method to get an instance of a class from a module
|
|
63
|
+
|
|
64
|
+
Args:
|
|
65
|
+
id (str): The name of the module and the class to be instantiated
|
|
66
|
+
filepath (str): The path to the module file
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
Any: An instance of the class if found, else None
|
|
70
|
+
"""
|
|
71
|
+
# Create the module specification
|
|
72
|
+
module_spec = create_module_spec(
|
|
73
|
+
id,
|
|
74
|
+
filepath,
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
# Check if the module specification exists
|
|
78
|
+
if module_spec:
|
|
79
|
+
# Import the module
|
|
80
|
+
module = import_module_from_spec(module_spec)
|
|
81
|
+
|
|
82
|
+
# Iterate through the attributes of the module
|
|
83
|
+
for attr in dir(module):
|
|
84
|
+
# Get the attribute object
|
|
85
|
+
obj = getattr(module, attr)
|
|
86
|
+
|
|
87
|
+
# Check if the attribute is a class and has the same module name as the id
|
|
88
|
+
if (
|
|
89
|
+
inspect.isclass(obj)
|
|
90
|
+
and "None" not in obj.__module__
|
|
91
|
+
and obj.__module__ == id
|
|
92
|
+
):
|
|
93
|
+
return obj
|
|
94
|
+
|
|
95
|
+
# Return None if no instance of the class is found
|
|
96
|
+
return None
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import time
|
|
2
|
+
from functools import wraps
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def timeit(func):
|
|
6
|
+
@wraps(func)
|
|
7
|
+
def timeit_wrapper(*args, **kwargs):
|
|
8
|
+
"""
|
|
9
|
+
A decorator function that measures the execution time of the decorated function.
|
|
10
|
+
|
|
11
|
+
Args:
|
|
12
|
+
*args: Variable length argument list.
|
|
13
|
+
**kwargs: Arbitrary keyword arguments.
|
|
14
|
+
|
|
15
|
+
Returns:
|
|
16
|
+
The result of the decorated function.
|
|
17
|
+
"""
|
|
18
|
+
start_time = time.perf_counter()
|
|
19
|
+
result = func(*args, **kwargs)
|
|
20
|
+
end_time = time.perf_counter()
|
|
21
|
+
total_time = end_time - start_time
|
|
22
|
+
print(f"[{func.__module__}] Running [{func.__name__}] took {total_time:.4f}s")
|
|
23
|
+
return result
|
|
24
|
+
|
|
25
|
+
return timeit_wrapper
|