ibm-watsonx-orchestrate 1.10.2__py3-none-any.whl → 1.11.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ibm_watsonx_orchestrate/__init__.py +2 -1
- ibm_watsonx_orchestrate/agent_builder/agents/types.py +13 -0
- ibm_watsonx_orchestrate/agent_builder/connections/types.py +53 -6
- ibm_watsonx_orchestrate/agent_builder/knowledge_bases/types.py +25 -10
- ibm_watsonx_orchestrate/cli/commands/agents/agents_command.py +10 -2
- ibm_watsonx_orchestrate/cli/commands/agents/agents_controller.py +404 -173
- ibm_watsonx_orchestrate/cli/commands/connections/connections_command.py +33 -4
- ibm_watsonx_orchestrate/cli/commands/connections/connections_controller.py +62 -6
- ibm_watsonx_orchestrate/cli/commands/copilot/copilot_controller.py +6 -2
- ibm_watsonx_orchestrate/cli/commands/environment/environment_command.py +1 -1
- ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py +174 -2
- ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py +93 -9
- ibm_watsonx_orchestrate/cli/commands/server/server_command.py +0 -3
- ibm_watsonx_orchestrate/cli/commands/server/types.py +15 -7
- ibm_watsonx_orchestrate/client/base_api_client.py +31 -10
- ibm_watsonx_orchestrate/client/connections/connections_client.py +14 -0
- ibm_watsonx_orchestrate/client/service_instance.py +19 -34
- ibm_watsonx_orchestrate/client/utils.py +3 -1
- ibm_watsonx_orchestrate/docker/compose-lite.yml +16 -11
- ibm_watsonx_orchestrate/docker/default.env +15 -13
- ibm_watsonx_orchestrate/flow_builder/data_map.py +4 -1
- ibm_watsonx_orchestrate/flow_builder/flows/flow.py +117 -7
- ibm_watsonx_orchestrate/flow_builder/node.py +76 -5
- ibm_watsonx_orchestrate/flow_builder/types.py +344 -10
- {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/METADATA +2 -2
- {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/RECORD +29 -29
- {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/WHEEL +0 -0
- {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/entry_points.txt +0 -0
- {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
import typer
|
2
2
|
from typing_extensions import Annotated, List
|
3
|
-
from ibm_watsonx_orchestrate.agent_builder.connections.types import ConnectionEnvironment, ConnectionPreference, ConnectionKind
|
3
|
+
from ibm_watsonx_orchestrate.agent_builder.connections.types import ConnectionEnvironment, ConnectionPreference, ConnectionKind, ConnectionCredentialsEntry
|
4
4
|
from ibm_watsonx_orchestrate.cli.commands.connections.connections_controller import (
|
5
5
|
add_connection,
|
6
6
|
remove_connection,
|
@@ -8,7 +8,9 @@ from ibm_watsonx_orchestrate.cli.commands.connections.connections_controller imp
|
|
8
8
|
import_connection,
|
9
9
|
configure_connection,
|
10
10
|
set_credentials_connection,
|
11
|
-
set_identity_provider_connection
|
11
|
+
set_identity_provider_connection,
|
12
|
+
token_entry_connection_credentials_parse,
|
13
|
+
auth_entry_connection_credentials_parse
|
12
14
|
)
|
13
15
|
|
14
16
|
connections_app = typer.Typer(no_args_is_help=True)
|
@@ -244,6 +246,22 @@ def set_credentials_connection_command(
|
|
244
246
|
help="For key_value, a key value pair in the form '<key>=<value>'. Multiple values can be passed using `-e key1=value1 -e key2=value2`"
|
245
247
|
)
|
246
248
|
] = None,
|
249
|
+
token_entries: Annotated[
|
250
|
+
List[ConnectionCredentialsEntry],
|
251
|
+
typer.Option(
|
252
|
+
'--token-entries', "-t",
|
253
|
+
parser=token_entry_connection_credentials_parse,
|
254
|
+
help="Custom field options for oauth types token request, a key value location option in the form 'location:<key>=<value>' or '<key>=<value>' with location defaulting to 'header'. Multiple values can be passed using `-t key1=value1 -t location:key2=value2`"
|
255
|
+
)
|
256
|
+
] = None,
|
257
|
+
auth_entries: Annotated[
|
258
|
+
List[ConnectionCredentialsEntry],
|
259
|
+
typer.Option(
|
260
|
+
'--auth-entries',
|
261
|
+
parser=auth_entry_connection_credentials_parse,
|
262
|
+
help="Custom field options for oauth_auth_code_flow auth server request, a key value location option in the form 'location:<key>=<value>' or '<key>=<value>' with location defaulting to 'query'. Note only 'query' is a valid location. Multiple values can be passed using `--auth-entries key1=value1 --auth-entries location:key2=value2`"
|
263
|
+
)
|
264
|
+
] = None,
|
247
265
|
):
|
248
266
|
set_credentials_connection(
|
249
267
|
app_id=app_id,
|
@@ -259,7 +277,9 @@ def set_credentials_connection_command(
|
|
259
277
|
auth_url=auth_url,
|
260
278
|
grant_type=grant_type,
|
261
279
|
scope=scope,
|
262
|
-
entries=entries
|
280
|
+
entries=entries,
|
281
|
+
token_entries=token_entries,
|
282
|
+
auth_entries=auth_entries
|
263
283
|
)
|
264
284
|
|
265
285
|
@connections_app.command(name="set-identity-provider")
|
@@ -311,6 +331,14 @@ def set_identity_provider_connection_command(
|
|
311
331
|
help='The grant-type of the the identity provider'
|
312
332
|
)
|
313
333
|
],
|
334
|
+
token_entries: Annotated[
|
335
|
+
List[ConnectionCredentialsEntry],
|
336
|
+
typer.Option(
|
337
|
+
'--token-entries', "-t",
|
338
|
+
parser=token_entry_connection_credentials_parse,
|
339
|
+
help="Custom field options for oauth types token request, a key value location option in the form 'location:<key>=<value>' or '<key>=<value>' with location defaulting to 'header'. Multiple values can be passed using `-t key1=value1 -t location:key2=value2`"
|
340
|
+
)
|
341
|
+
] = None,
|
314
342
|
):
|
315
343
|
set_identity_provider_connection(
|
316
344
|
app_id=app_id,
|
@@ -319,5 +347,6 @@ def set_identity_provider_connection_command(
|
|
319
347
|
client_id=client_id,
|
320
348
|
client_secret=client_secret,
|
321
349
|
scope=scope,
|
322
|
-
grant_type=grant_type
|
350
|
+
grant_type=grant_type,
|
351
|
+
token_entries=token_entries
|
323
352
|
)
|
@@ -27,7 +27,10 @@ from ibm_watsonx_orchestrate.agent_builder.connections.types import (
|
|
27
27
|
KeyValueConnectionCredentials,
|
28
28
|
CREDENTIALS,
|
29
29
|
IdentityProviderCredentials,
|
30
|
-
OAUTH_CONNECTION_TYPES
|
30
|
+
OAUTH_CONNECTION_TYPES,
|
31
|
+
ConnectionCredentialsEntryLocation,
|
32
|
+
ConnectionCredentialsEntry,
|
33
|
+
ConnectionCredentialsCustomFields
|
31
34
|
|
32
35
|
)
|
33
36
|
|
@@ -167,6 +170,13 @@ def _validate_connection_params(type: ConnectionType, **args) -> None:
|
|
167
170
|
f"Missing flags --grant-type is required for type {type}"
|
168
171
|
)
|
169
172
|
|
173
|
+
if type != ConnectionType.OAUTH2_AUTH_CODE and (
|
174
|
+
args.get('auth_entries')
|
175
|
+
):
|
176
|
+
raise typer.BadParameter(
|
177
|
+
f"The flag --auth-entries is only supported by type {type}"
|
178
|
+
)
|
179
|
+
|
170
180
|
|
171
181
|
def _parse_entry(entry: str) -> dict[str,str]:
|
172
182
|
split_entry = entry.split('=', 1)
|
@@ -176,6 +186,19 @@ def _parse_entry(entry: str) -> dict[str,str]:
|
|
176
186
|
exit(1)
|
177
187
|
return {split_entry[0]: split_entry[1]}
|
178
188
|
|
189
|
+
def _get_oauth_custom_fields(token_entries: List[ConnectionCredentialsEntry] | None, auth_entries: List[ConnectionCredentialsEntry] | None) -> dict:
|
190
|
+
custom_fields = ConnectionCredentialsCustomFields()
|
191
|
+
|
192
|
+
if token_entries:
|
193
|
+
for entry in token_entries:
|
194
|
+
custom_fields.add_field(entry, is_token=True)
|
195
|
+
|
196
|
+
if auth_entries:
|
197
|
+
for entry in auth_entries:
|
198
|
+
custom_fields.add_field(entry, is_token=False)
|
199
|
+
|
200
|
+
return custom_fields.model_dump(exclude_none=True)
|
201
|
+
|
179
202
|
def _get_credentials(type: ConnectionType, **kwargs):
|
180
203
|
match type:
|
181
204
|
case ConnectionType.BASIC_AUTH:
|
@@ -192,18 +215,21 @@ def _get_credentials(type: ConnectionType, **kwargs):
|
|
192
215
|
api_key=kwargs.get("api_key")
|
193
216
|
)
|
194
217
|
case ConnectionType.OAUTH2_AUTH_CODE:
|
218
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
195
219
|
return OAuth2AuthCodeCredentials(
|
196
220
|
authorization_url=kwargs.get("auth_url"),
|
197
221
|
client_id=kwargs.get("client_id"),
|
198
222
|
client_secret=kwargs.get("client_secret"),
|
199
223
|
token_url=kwargs.get("token_url"),
|
200
|
-
scope=kwargs.get("scope")
|
224
|
+
scope=kwargs.get("scope"),
|
225
|
+
**custom_fields
|
201
226
|
)
|
202
227
|
case ConnectionType.OAUTH2_CLIENT_CREDS:
|
203
228
|
# using filtered args as default values will not be set if 'None' is passed, causing validation errors
|
204
229
|
keys = ["client_id","client_secret","token_url","grant_type","send_via", "scope"]
|
205
230
|
filtered_args = { key_name: kwargs[key_name] for key_name in keys if kwargs.get(key_name) }
|
206
|
-
|
231
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
232
|
+
return OAuth2ClientCredentials(**filtered_args, **custom_fields)
|
207
233
|
# case ConnectionType.OAUTH2_IMPLICIT:
|
208
234
|
# return OAuth2ImplicitCredentials(
|
209
235
|
# authorization_url=kwargs.get("auth_url"),
|
@@ -212,13 +238,16 @@ def _get_credentials(type: ConnectionType, **kwargs):
|
|
212
238
|
case ConnectionType.OAUTH2_PASSWORD:
|
213
239
|
keys = ["username", "password", "client_id","client_secret","token_url","grant_type", "scope"]
|
214
240
|
filtered_args = { key_name: kwargs[key_name] for key_name in keys if kwargs.get(key_name) }
|
215
|
-
|
241
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
242
|
+
return OAuth2PasswordCredentials(**filtered_args, **custom_fields)
|
216
243
|
|
217
244
|
case ConnectionType.OAUTH_ON_BEHALF_OF_FLOW:
|
245
|
+
custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
|
218
246
|
return OAuthOnBehalfOfCredentials(
|
219
247
|
client_id=kwargs.get("client_id"),
|
220
248
|
access_token_url=kwargs.get("token_url"),
|
221
|
-
grant_type=kwargs.get("grant_type")
|
249
|
+
grant_type=kwargs.get("grant_type"),
|
250
|
+
**custom_fields
|
222
251
|
)
|
223
252
|
case ConnectionType.KEY_VALUE:
|
224
253
|
env = {}
|
@@ -231,6 +260,23 @@ def _get_credentials(type: ConnectionType, **kwargs):
|
|
231
260
|
case _:
|
232
261
|
raise ValueError(f"Invalid type '{type}' selected")
|
233
262
|
|
263
|
+
def _connection_credentials_parse_entry(text: str, default_location: ConnectionCredentialsEntryLocation) -> ConnectionCredentialsEntry:
|
264
|
+
location_kv_pair = text.split(":", 1)
|
265
|
+
key_value = location_kv_pair[-1]
|
266
|
+
location = location_kv_pair[0] if len(location_kv_pair)>1 else default_location
|
267
|
+
|
268
|
+
valid_locations = [item.value for item in ConnectionCredentialsEntryLocation]
|
269
|
+
if location not in valid_locations:
|
270
|
+
raise typer.BadParameter(f"The provided location '{location}' is not in the allowed values {valid_locations}.")
|
271
|
+
|
272
|
+
key_value_pair = key_value.split('=', 1)
|
273
|
+
if len(key_value_pair) != 2:
|
274
|
+
message = f"The entry '{text}' is not in the expected form '<location>:<key>=<value>' or '<key>=<value>'"
|
275
|
+
raise typer.BadParameter(message)
|
276
|
+
key, value = key_value_pair[0], key_value_pair[1]
|
277
|
+
|
278
|
+
return ConnectionCredentialsEntry(key=key, value=value, location=location)
|
279
|
+
|
234
280
|
|
235
281
|
def add_configuration(config: ConnectionConfiguration) -> None:
|
236
282
|
client = get_connections_client()
|
@@ -524,5 +570,15 @@ def set_identity_provider_connection(
|
|
524
570
|
logger.error(f"Cannot set Identity Provider when 'sso' is false in configuration. Please enable sso for connection '{app_id}' in environment '{environment}' and try again.")
|
525
571
|
sys.exit(1)
|
526
572
|
|
527
|
-
|
573
|
+
custom_fields = _get_oauth_custom_fields(token_entries=kwargs.get("token_entries"), auth_entries=None)
|
574
|
+
idp = IdentityProviderCredentials(**kwargs, **custom_fields)
|
528
575
|
add_identity_provider(app_id=app_id, environment=environment, idp=idp)
|
576
|
+
|
577
|
+
def token_entry_connection_credentials_parse(text: str) -> ConnectionCredentialsEntry:
|
578
|
+
return _connection_credentials_parse_entry(text=text, default_location=ConnectionCredentialsEntryLocation.HEADER)
|
579
|
+
|
580
|
+
def auth_entry_connection_credentials_parse(text: str) -> ConnectionCredentialsEntry:
|
581
|
+
entry = _connection_credentials_parse_entry(text=text, default_location=ConnectionCredentialsEntryLocation.QUERY)
|
582
|
+
if entry.location != ConnectionCredentialsEntryLocation.QUERY:
|
583
|
+
raise typer.BadParameter(f"Only location '{ConnectionCredentialsEntryLocation.QUERY}' is supported for --auth-entry")
|
584
|
+
return entry
|
@@ -327,7 +327,11 @@ def talk_to_cpe(cpe_client, samples_file=None, context_data=None):
|
|
327
327
|
|
328
328
|
|
329
329
|
def prompt_tune(agent_spec: str, output_file: str | None, samples_file: str | None, dry_run_flag: bool) -> None:
|
330
|
-
|
330
|
+
agents = AgentsController.import_agent(file=agent_spec, app_id=None)
|
331
|
+
if not agents:
|
332
|
+
logger.error("Invalid agent spec file provided, no agent found.")
|
333
|
+
sys.exit(1)
|
334
|
+
agent = agents[0]
|
331
335
|
agent_kind = agent.kind
|
332
336
|
|
333
337
|
if agent_kind != AgentKind.NATIVE:
|
@@ -342,7 +346,7 @@ def prompt_tune(agent_spec: str, output_file: str | None, samples_file: str | No
|
|
342
346
|
|
343
347
|
client = get_cpe_client()
|
344
348
|
|
345
|
-
instr = agent.instructions
|
349
|
+
instr = agent.instructions if agent.instructions else ""
|
346
350
|
|
347
351
|
tools = _get_tools_from_names(agent.tools)
|
348
352
|
|
@@ -70,7 +70,7 @@ def add_env(
|
|
70
70
|
] = None,
|
71
71
|
type: Annotated[
|
72
72
|
EnvironmentAuthType,
|
73
|
-
typer.Option("--type", "-t", help="The type of auth you wish to use"),
|
73
|
+
typer.Option("--type", "-t", help="The type of auth you wish to use. This overrides the auth type that is inferred from the url"),
|
74
74
|
] = None,
|
75
75
|
insecure: Annotated[
|
76
76
|
bool,
|
@@ -16,7 +16,7 @@ from typing import Optional
|
|
16
16
|
from typing_extensions import Annotated
|
17
17
|
|
18
18
|
from ibm_watsonx_orchestrate import __version__
|
19
|
-
from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
|
19
|
+
from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController, EvaluateMode
|
20
20
|
from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
|
21
21
|
|
22
22
|
logger = logging.getLogger(__name__)
|
@@ -220,6 +220,13 @@ def analyze(data_path: Annotated[
|
|
220
220
|
help="Path to the directory that has the saved results"
|
221
221
|
)
|
222
222
|
],
|
223
|
+
tool_definition_path: Annotated[
|
224
|
+
Optional[str],
|
225
|
+
typer.Option(
|
226
|
+
"--tools-path", "-t",
|
227
|
+
help="Path to the directory containing tool definitions."
|
228
|
+
)
|
229
|
+
] = None,
|
223
230
|
user_env_file: Annotated[
|
224
231
|
Optional[str],
|
225
232
|
typer.Option(
|
@@ -230,7 +237,10 @@ def analyze(data_path: Annotated[
|
|
230
237
|
|
231
238
|
validate_watsonx_credentials(user_env_file)
|
232
239
|
controller = EvaluationsController()
|
233
|
-
controller.analyze(
|
240
|
+
controller.analyze(
|
241
|
+
data_path=data_path,
|
242
|
+
tool_definition_path=tool_definition_path
|
243
|
+
)
|
234
244
|
|
235
245
|
@evaluation_app.command(name="validate-external", help="Validate an external agent against a set of inputs")
|
236
246
|
def validate_external(
|
@@ -375,3 +385,165 @@ def validate_external(
|
|
375
385
|
msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"
|
376
386
|
|
377
387
|
rich.print(Panel(msg))
|
388
|
+
|
389
|
+
@evaluation_app.command(name="quick-eval",
|
390
|
+
short_help="Evaluate agent against a suite of static metrics and LLM-as-a-judge metrics",
|
391
|
+
help="""
|
392
|
+
Use the quick-eval command to evaluate your agent against a suite of static metrics and LLM-as-a-judge metrics.
|
393
|
+
""")
|
394
|
+
def quick_eval(
|
395
|
+
config_file: Annotated[
|
396
|
+
Optional[str],
|
397
|
+
typer.Option(
|
398
|
+
"--config", "-c",
|
399
|
+
help="Path to YAML configuration file containing evaluation settings."
|
400
|
+
)
|
401
|
+
] = None,
|
402
|
+
test_paths: Annotated[
|
403
|
+
Optional[str],
|
404
|
+
typer.Option(
|
405
|
+
"--test-paths", "-p",
|
406
|
+
help="Paths to the test files and/or directories to evaluate, separated by commas."
|
407
|
+
),
|
408
|
+
] = None,
|
409
|
+
tools_path: Annotated[
|
410
|
+
str,
|
411
|
+
typer.Option(
|
412
|
+
"--tools-path", "-t",
|
413
|
+
help="Path to the directory containing tool definitions."
|
414
|
+
)
|
415
|
+
] = None,
|
416
|
+
output_dir: Annotated[
|
417
|
+
Optional[str],
|
418
|
+
typer.Option(
|
419
|
+
"--output-dir", "-o",
|
420
|
+
help="Directory to save the evaluation results."
|
421
|
+
)
|
422
|
+
] = None,
|
423
|
+
user_env_file: Annotated[
|
424
|
+
Optional[str],
|
425
|
+
typer.Option(
|
426
|
+
"--env-file", "-e",
|
427
|
+
help="Path to a .env file that overrides default.env. Then environment variables override both."
|
428
|
+
),
|
429
|
+
] = None
|
430
|
+
):
|
431
|
+
if not config_file:
|
432
|
+
if not test_paths or not output_dir:
|
433
|
+
logger.error("Error: Both --test-paths and --output-dir must be provided when not using a config file")
|
434
|
+
exit(1)
|
435
|
+
|
436
|
+
validate_watsonx_credentials(user_env_file)
|
437
|
+
|
438
|
+
if tools_path is None:
|
439
|
+
logger.error("When running `quick-eval`, please provide the path to your tools file.")
|
440
|
+
sys.exit(1)
|
441
|
+
|
442
|
+
controller = EvaluationsController()
|
443
|
+
controller.evaluate(
|
444
|
+
config_file=config_file,
|
445
|
+
test_paths=test_paths,
|
446
|
+
output_dir=output_dir,
|
447
|
+
tools_path=tools_path, mode=EvaluateMode.referenceless
|
448
|
+
)
|
449
|
+
|
450
|
+
|
451
|
+
red_teaming_app = typer.Typer(no_args_is_help=True)
|
452
|
+
evaluation_app.add_typer(red_teaming_app, name="red-teaming")
|
453
|
+
|
454
|
+
|
455
|
+
@red_teaming_app.command("list", help="List available red-teaming attack plans")
|
456
|
+
def list_plans():
|
457
|
+
controller = EvaluationsController()
|
458
|
+
controller.list_red_teaming_attacks()
|
459
|
+
|
460
|
+
|
461
|
+
@red_teaming_app.command("plan", help="Generate red-teaming attacks")
|
462
|
+
def plan(
|
463
|
+
attacks_list: Annotated[
|
464
|
+
str,
|
465
|
+
typer.Option(
|
466
|
+
"--attacks-list",
|
467
|
+
"-a",
|
468
|
+
help="Comma-separated list of red-teaming attacks to generate.",
|
469
|
+
),
|
470
|
+
],
|
471
|
+
datasets_path: Annotated[
|
472
|
+
str,
|
473
|
+
typer.Option(
|
474
|
+
"--datasets-path",
|
475
|
+
"-d",
|
476
|
+
help="Path to datasets for red-teaming. This can also be a comma-separated list of files or directories.",
|
477
|
+
),
|
478
|
+
],
|
479
|
+
agents_path: Annotated[
|
480
|
+
str, typer.Option("--agents-path", "-g", help="Path to the directory containing all agent definitions.")
|
481
|
+
],
|
482
|
+
target_agent_name: Annotated[
|
483
|
+
str,
|
484
|
+
typer.Option(
|
485
|
+
"--target-agent-name",
|
486
|
+
"-t",
|
487
|
+
help="Name of the target agent to attack (should be present in agents-path).",
|
488
|
+
),
|
489
|
+
],
|
490
|
+
output_dir: Annotated[
|
491
|
+
Optional[str],
|
492
|
+
typer.Option("--output-dir", "-o", help="Directory to save generated attacks.")
|
493
|
+
]=None,
|
494
|
+
user_env_file: Annotated[
|
495
|
+
Optional[str],
|
496
|
+
typer.Option(
|
497
|
+
"--env-file",
|
498
|
+
"-e",
|
499
|
+
help="Path to a .env file that overrides default.env. Then environment variables override both.",
|
500
|
+
),
|
501
|
+
] = None,
|
502
|
+
max_variants: Annotated[
|
503
|
+
Optional[int],
|
504
|
+
typer.Option(
|
505
|
+
"--max_variants",
|
506
|
+
"-n",
|
507
|
+
help="Number of variants to generate per attack type.",
|
508
|
+
),
|
509
|
+
] = None,
|
510
|
+
|
511
|
+
):
|
512
|
+
validate_watsonx_credentials(user_env_file)
|
513
|
+
controller = EvaluationsController()
|
514
|
+
controller.generate_red_teaming_attacks(
|
515
|
+
attacks_list=attacks_list,
|
516
|
+
datasets_path=datasets_path,
|
517
|
+
agents_path=agents_path,
|
518
|
+
target_agent_name=target_agent_name,
|
519
|
+
output_dir=output_dir,
|
520
|
+
max_variants=max_variants,
|
521
|
+
)
|
522
|
+
|
523
|
+
|
524
|
+
@red_teaming_app.command("run", help="Run red-teaming attacks")
|
525
|
+
def run(
|
526
|
+
attack_paths: Annotated[
|
527
|
+
str,
|
528
|
+
typer.Option(
|
529
|
+
"--attack-paths",
|
530
|
+
"-a",
|
531
|
+
help="Path or list of paths (comma-separated) to directories containing attack files.",
|
532
|
+
),
|
533
|
+
],
|
534
|
+
output_dir: Annotated[
|
535
|
+
Optional[str],
|
536
|
+
typer.Option("--output-dir", "-o", help="Directory to save attack results."),
|
537
|
+
] = None,
|
538
|
+
user_env_file: Annotated[
|
539
|
+
Optional[str],
|
540
|
+
typer.Option(
|
541
|
+
"--env-file",
|
542
|
+
"-e",
|
543
|
+
help="Path to a .env file that overrides default.env. Then environment variables override both.",
|
544
|
+
),
|
545
|
+
] = None,
|
546
|
+
):
|
547
|
+
validate_watsonx_credentials(user_env_file)
|
548
|
+
controller = EvaluationsController()
|
549
|
+
controller.run_red_teaming_attacks(attack_paths=attack_paths, output_dir=output_dir)
|
@@ -1,17 +1,23 @@
|
|
1
1
|
import logging
|
2
2
|
import os.path
|
3
3
|
from typing import List, Dict, Optional, Tuple
|
4
|
+
from enum import StrEnum
|
4
5
|
import csv
|
5
6
|
from pathlib import Path
|
6
7
|
import sys
|
7
8
|
from wxo_agentic_evaluation import main as evaluate
|
9
|
+
from wxo_agentic_evaluation import quick_eval
|
8
10
|
from wxo_agentic_evaluation.tool_planner import build_snapshot
|
9
|
-
from wxo_agentic_evaluation.analyze_run import
|
11
|
+
from wxo_agentic_evaluation.analyze_run import Analyzer
|
10
12
|
from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stories
|
11
|
-
from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig, ProviderConfig
|
13
|
+
from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig, ProviderConfig, AttackConfig, QuickEvalConfig
|
12
14
|
from wxo_agentic_evaluation.record_chat import record_chats
|
13
15
|
from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
|
14
16
|
from wxo_agentic_evaluation.external_agent.performance_test import ExternalAgentPerformanceTest
|
17
|
+
from wxo_agentic_evaluation.red_teaming.attack_list import print_attacks
|
18
|
+
from wxo_agentic_evaluation.red_teaming import attack_generator
|
19
|
+
from wxo_agentic_evaluation.red_teaming.attack_runner import run_attacks
|
20
|
+
from wxo_agentic_evaluation.arg_configs import AttackGeneratorConfig
|
15
21
|
from ibm_watsonx_orchestrate import __version__
|
16
22
|
from ibm_watsonx_orchestrate.cli.config import Config, ENV_WXO_URL_OPT, AUTH_CONFIG_FILE, AUTH_CONFIG_FILE_FOLDER, AUTH_SECTION_HEADER, AUTH_MCSP_TOKEN_OPT
|
17
23
|
from ibm_watsonx_orchestrate.utils.utils import yaml_safe_load
|
@@ -21,6 +27,9 @@ import uuid
|
|
21
27
|
|
22
28
|
logger = logging.getLogger(__name__)
|
23
29
|
|
30
|
+
class EvaluateMode(StrEnum):
|
31
|
+
default = "default" # referenceFUL evaluation
|
32
|
+
referenceless = "referenceless"
|
24
33
|
|
25
34
|
class EvaluationsController:
|
26
35
|
def __init__(self):
|
@@ -38,7 +47,7 @@ class EvaluationsController:
|
|
38
47
|
|
39
48
|
return url, tenant_name, token
|
40
49
|
|
41
|
-
def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None) -> None:
|
50
|
+
def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None, tools_path: str = None, mode: str = EvaluateMode.default) -> None:
|
42
51
|
url, tenant_name, token = self._get_env_config()
|
43
52
|
|
44
53
|
if "WATSONX_SPACE_ID" in os.environ and "WATSONX_APIKEY" in os.environ:
|
@@ -90,9 +99,13 @@ class EvaluationsController:
|
|
90
99
|
config_data["output_dir"] = output_dir
|
91
100
|
logger.info(f"Using output directory: {config_data['output_dir']}")
|
92
101
|
|
93
|
-
|
94
|
-
|
95
|
-
|
102
|
+
if mode == EvaluateMode.default:
|
103
|
+
config = TestConfig(**config_data)
|
104
|
+
evaluate.main(config)
|
105
|
+
elif mode == EvaluateMode.referenceless:
|
106
|
+
config_data["tools_path"] = tools_path
|
107
|
+
config = QuickEvalConfig(**config_data)
|
108
|
+
quick_eval.main(config)
|
96
109
|
|
97
110
|
def record(self, output_dir) -> None:
|
98
111
|
|
@@ -160,9 +173,13 @@ class EvaluationsController:
|
|
160
173
|
|
161
174
|
logger.info("Test cases stored at: %s", output_dir)
|
162
175
|
|
163
|
-
def analyze(self, data_path: str) -> None:
|
164
|
-
config = AnalyzeConfig(
|
165
|
-
|
176
|
+
def analyze(self, data_path: str, tool_definition_path: str) -> None:
|
177
|
+
config = AnalyzeConfig(
|
178
|
+
data_path=data_path,
|
179
|
+
tool_definition_path=tool_definition_path
|
180
|
+
)
|
181
|
+
analyzer = Analyzer()
|
182
|
+
analyzer.analyze(config)
|
166
183
|
|
167
184
|
def summarize(self) -> None:
|
168
185
|
pass
|
@@ -187,3 +204,70 @@ class EvaluationsController:
|
|
187
204
|
generated_performance_tests = performance_test.generate_tests()
|
188
205
|
|
189
206
|
return generated_performance_tests
|
207
|
+
|
208
|
+
def list_red_teaming_attacks(self):
|
209
|
+
print_attacks()
|
210
|
+
|
211
|
+
def generate_red_teaming_attacks(
|
212
|
+
self,
|
213
|
+
attacks_list: str,
|
214
|
+
datasets_path: str,
|
215
|
+
agents_path: str,
|
216
|
+
target_agent_name: str,
|
217
|
+
output_dir: Optional[str] = None,
|
218
|
+
max_variants: Optional[int] = None,
|
219
|
+
):
|
220
|
+
if output_dir is None:
|
221
|
+
output_dir = os.path.join(os.getcwd(), "red_teaming_attacks")
|
222
|
+
os.makedirs(output_dir, exist_ok=True)
|
223
|
+
logger.info(f"No output directory specified. Using default: {output_dir}")
|
224
|
+
|
225
|
+
results = attack_generator.main(
|
226
|
+
AttackGeneratorConfig(
|
227
|
+
attacks_list=attacks_list.split(","),
|
228
|
+
datasets_path=datasets_path.split(","),
|
229
|
+
agents_path=agents_path,
|
230
|
+
target_agent_name=target_agent_name,
|
231
|
+
output_dir=output_dir,
|
232
|
+
max_variants=max_variants,
|
233
|
+
)
|
234
|
+
)
|
235
|
+
logger.info(f"Generated {len(results)} attacks and saved to {output_dir}")
|
236
|
+
|
237
|
+
def run_red_teaming_attacks(self, attack_paths: str, output_dir: Optional[str] = None) -> None:
|
238
|
+
url, tenant_name, token = self._get_env_config()
|
239
|
+
|
240
|
+
if "WATSONX_SPACE_ID" in os.environ and "WATSONX_APIKEY" in os.environ:
|
241
|
+
provider = "watsonx"
|
242
|
+
elif "WO_INSTANCE" in os.environ and "WO_API_KEY" in os.environ:
|
243
|
+
provider = "model_proxy"
|
244
|
+
else:
|
245
|
+
logger.error(
|
246
|
+
"No provider found. Please either provide a config_file or set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables."
|
247
|
+
)
|
248
|
+
sys.exit(1)
|
249
|
+
|
250
|
+
config_data = {
|
251
|
+
"auth_config": AuthConfig(
|
252
|
+
url=url,
|
253
|
+
tenant_name=tenant_name,
|
254
|
+
token=token,
|
255
|
+
),
|
256
|
+
"provider_config": ProviderConfig(
|
257
|
+
provider=provider,
|
258
|
+
model_id="meta-llama/llama-3-405b-instruct",
|
259
|
+
),
|
260
|
+
}
|
261
|
+
|
262
|
+
config_data["attack_paths"] = attack_paths.split(",")
|
263
|
+
if output_dir:
|
264
|
+
config_data["output_dir"] = output_dir
|
265
|
+
else:
|
266
|
+
config_data["output_dir"] = os.path.join(os.getcwd(), "red_teaming_results")
|
267
|
+
os.makedirs(config_data["output_dir"], exist_ok=True)
|
268
|
+
logger.info(f"No output directory specified. Using default: {config_data['output_dir']}")
|
269
|
+
|
270
|
+
|
271
|
+
config = AttackConfig(**config_data)
|
272
|
+
|
273
|
+
run_attacks(config)
|
@@ -191,9 +191,6 @@ def get_default_registry_env_vars_by_dev_edition_source(default_env: dict, user_
|
|
191
191
|
parsed = urlparse(wo_url)
|
192
192
|
hostname = parsed.hostname
|
193
193
|
|
194
|
-
if not hostname or not hostname.startswith("api."):
|
195
|
-
raise ValueError(f"Invalid WO_INSTANCE URL: '{wo_url}'. It should starts with 'api.'")
|
196
|
-
|
197
194
|
registry_url = f"registry.{hostname[4:]}/cp/wxo-lite"
|
198
195
|
else:
|
199
196
|
raise ValueError(f"Unknown value for developer edition source: {source}. Must be one of ['internal', 'myibm', 'orchestrate'].")
|
@@ -1,5 +1,6 @@
|
|
1
1
|
import logging
|
2
2
|
import sys
|
3
|
+
import uuid
|
3
4
|
from enum import Enum
|
4
5
|
from pydantic import BaseModel, model_validator, ConfigDict
|
5
6
|
|
@@ -43,9 +44,6 @@ class WatsonXAIEnvConfig(BaseModel):
|
|
43
44
|
if not config.get("WATSONX_SPACE_ID") and not config.get("WATSONX_APIKEY"):
|
44
45
|
raise ValueError("Missing configuration requirements 'WATSONX_SPACE_ID' and 'WATSONX_APIKEY'")
|
45
46
|
|
46
|
-
if config.get("WATSONX_SPACE_ID") and not config.get("WATSONX_APIKEY"):
|
47
|
-
logger.error("Cannot use env var 'WATSONX_SPACE_ID' without setting the corresponding 'WATSONX_APIKEY'")
|
48
|
-
sys.exit(1)
|
49
47
|
|
50
48
|
if not config.get("WATSONX_SPACE_ID") and config.get("WATSONX_APIKEY"):
|
51
49
|
logger.error("Cannot use env var 'WATSONX_APIKEY' without setting the corresponding 'WATSONX_SPACE_ID'")
|
@@ -54,6 +52,12 @@ class WatsonXAIEnvConfig(BaseModel):
|
|
54
52
|
config["USE_SAAS_ML_TOOLS_RUNTIME"] = False
|
55
53
|
return config
|
56
54
|
|
55
|
+
def is_valid_uuid(value) -> bool:
|
56
|
+
try:
|
57
|
+
uuid.UUID(str(value))
|
58
|
+
return True
|
59
|
+
except (ValueError, TypeError, AttributeError):
|
60
|
+
return False
|
57
61
|
|
58
62
|
class ModelGatewayEnvConfig(BaseModel):
|
59
63
|
WO_API_KEY: str | None = None
|
@@ -84,8 +88,11 @@ class ModelGatewayEnvConfig(BaseModel):
|
|
84
88
|
if not config.get("AUTHORIZATION_URL"):
|
85
89
|
inferred_auth_url = AUTH_TYPE_DEFAULT_URL_MAPPING.get(auth_type)
|
86
90
|
if not inferred_auth_url:
|
87
|
-
|
88
|
-
|
91
|
+
if auth_type == WoAuthType.CPD:
|
92
|
+
inferred_auth_url = config.get("WO_INSTANCE") + '/icp4d-api/v1/authorize'
|
93
|
+
else:
|
94
|
+
logger.error(f"No 'AUTHORIZATION_URL' found. Auth type '{auth_type}' does not support defaulting. Please set the 'AUTHORIZATION_URL' explictly")
|
95
|
+
sys.exit(1)
|
89
96
|
config["AUTHORIZATION_URL"] = inferred_auth_url
|
90
97
|
|
91
98
|
if auth_type != WoAuthType.CPD:
|
@@ -101,6 +108,7 @@ class ModelGatewayEnvConfig(BaseModel):
|
|
101
108
|
sys.exit(1)
|
102
109
|
|
103
110
|
config["USE_SAAS_ML_TOOLS_RUNTIME"] = True
|
104
|
-
|
105
|
-
|
111
|
+
if not is_valid_uuid(config.get("WATSONX_SPACE_ID")):
|
112
|
+
# Fake (but valid) UUIDv4 for knowledgebase check
|
113
|
+
config["WATSONX_SPACE_ID"] = "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa"
|
106
114
|
return config
|