ibm-watsonx-orchestrate 1.10.2__py3-none-any.whl → 1.11.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (29) hide show
  1. ibm_watsonx_orchestrate/__init__.py +2 -1
  2. ibm_watsonx_orchestrate/agent_builder/agents/types.py +13 -0
  3. ibm_watsonx_orchestrate/agent_builder/connections/types.py +53 -6
  4. ibm_watsonx_orchestrate/agent_builder/knowledge_bases/types.py +25 -10
  5. ibm_watsonx_orchestrate/cli/commands/agents/agents_command.py +10 -2
  6. ibm_watsonx_orchestrate/cli/commands/agents/agents_controller.py +404 -173
  7. ibm_watsonx_orchestrate/cli/commands/connections/connections_command.py +33 -4
  8. ibm_watsonx_orchestrate/cli/commands/connections/connections_controller.py +62 -6
  9. ibm_watsonx_orchestrate/cli/commands/copilot/copilot_controller.py +6 -2
  10. ibm_watsonx_orchestrate/cli/commands/environment/environment_command.py +1 -1
  11. ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_command.py +174 -2
  12. ibm_watsonx_orchestrate/cli/commands/evaluations/evaluations_controller.py +93 -9
  13. ibm_watsonx_orchestrate/cli/commands/server/server_command.py +0 -3
  14. ibm_watsonx_orchestrate/cli/commands/server/types.py +15 -7
  15. ibm_watsonx_orchestrate/client/base_api_client.py +31 -10
  16. ibm_watsonx_orchestrate/client/connections/connections_client.py +14 -0
  17. ibm_watsonx_orchestrate/client/service_instance.py +19 -34
  18. ibm_watsonx_orchestrate/client/utils.py +3 -1
  19. ibm_watsonx_orchestrate/docker/compose-lite.yml +16 -11
  20. ibm_watsonx_orchestrate/docker/default.env +15 -13
  21. ibm_watsonx_orchestrate/flow_builder/data_map.py +4 -1
  22. ibm_watsonx_orchestrate/flow_builder/flows/flow.py +117 -7
  23. ibm_watsonx_orchestrate/flow_builder/node.py +76 -5
  24. ibm_watsonx_orchestrate/flow_builder/types.py +344 -10
  25. {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/METADATA +2 -2
  26. {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/RECORD +29 -29
  27. {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/WHEEL +0 -0
  28. {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/entry_points.txt +0 -0
  29. {ibm_watsonx_orchestrate-1.10.2.dist-info → ibm_watsonx_orchestrate-1.11.0.dist-info}/licenses/LICENSE +0 -0
@@ -1,6 +1,6 @@
1
1
  import typer
2
2
  from typing_extensions import Annotated, List
3
- from ibm_watsonx_orchestrate.agent_builder.connections.types import ConnectionEnvironment, ConnectionPreference, ConnectionKind
3
+ from ibm_watsonx_orchestrate.agent_builder.connections.types import ConnectionEnvironment, ConnectionPreference, ConnectionKind, ConnectionCredentialsEntry
4
4
  from ibm_watsonx_orchestrate.cli.commands.connections.connections_controller import (
5
5
  add_connection,
6
6
  remove_connection,
@@ -8,7 +8,9 @@ from ibm_watsonx_orchestrate.cli.commands.connections.connections_controller imp
8
8
  import_connection,
9
9
  configure_connection,
10
10
  set_credentials_connection,
11
- set_identity_provider_connection
11
+ set_identity_provider_connection,
12
+ token_entry_connection_credentials_parse,
13
+ auth_entry_connection_credentials_parse
12
14
  )
13
15
 
14
16
  connections_app = typer.Typer(no_args_is_help=True)
@@ -244,6 +246,22 @@ def set_credentials_connection_command(
244
246
  help="For key_value, a key value pair in the form '<key>=<value>'. Multiple values can be passed using `-e key1=value1 -e key2=value2`"
245
247
  )
246
248
  ] = None,
249
+ token_entries: Annotated[
250
+ List[ConnectionCredentialsEntry],
251
+ typer.Option(
252
+ '--token-entries', "-t",
253
+ parser=token_entry_connection_credentials_parse,
254
+ help="Custom field options for oauth types token request, a key value location option in the form 'location:<key>=<value>' or '<key>=<value>' with location defaulting to 'header'. Multiple values can be passed using `-t key1=value1 -t location:key2=value2`"
255
+ )
256
+ ] = None,
257
+ auth_entries: Annotated[
258
+ List[ConnectionCredentialsEntry],
259
+ typer.Option(
260
+ '--auth-entries',
261
+ parser=auth_entry_connection_credentials_parse,
262
+ help="Custom field options for oauth_auth_code_flow auth server request, a key value location option in the form 'location:<key>=<value>' or '<key>=<value>' with location defaulting to 'query'. Note only 'query' is a valid location. Multiple values can be passed using `--auth-entries key1=value1 --auth-entries location:key2=value2`"
263
+ )
264
+ ] = None,
247
265
  ):
248
266
  set_credentials_connection(
249
267
  app_id=app_id,
@@ -259,7 +277,9 @@ def set_credentials_connection_command(
259
277
  auth_url=auth_url,
260
278
  grant_type=grant_type,
261
279
  scope=scope,
262
- entries=entries
280
+ entries=entries,
281
+ token_entries=token_entries,
282
+ auth_entries=auth_entries
263
283
  )
264
284
 
265
285
  @connections_app.command(name="set-identity-provider")
@@ -311,6 +331,14 @@ def set_identity_provider_connection_command(
311
331
  help='The grant-type of the the identity provider'
312
332
  )
313
333
  ],
334
+ token_entries: Annotated[
335
+ List[ConnectionCredentialsEntry],
336
+ typer.Option(
337
+ '--token-entries', "-t",
338
+ parser=token_entry_connection_credentials_parse,
339
+ help="Custom field options for oauth types token request, a key value location option in the form 'location:<key>=<value>' or '<key>=<value>' with location defaulting to 'header'. Multiple values can be passed using `-t key1=value1 -t location:key2=value2`"
340
+ )
341
+ ] = None,
314
342
  ):
315
343
  set_identity_provider_connection(
316
344
  app_id=app_id,
@@ -319,5 +347,6 @@ def set_identity_provider_connection_command(
319
347
  client_id=client_id,
320
348
  client_secret=client_secret,
321
349
  scope=scope,
322
- grant_type=grant_type
350
+ grant_type=grant_type,
351
+ token_entries=token_entries
323
352
  )
@@ -27,7 +27,10 @@ from ibm_watsonx_orchestrate.agent_builder.connections.types import (
27
27
  KeyValueConnectionCredentials,
28
28
  CREDENTIALS,
29
29
  IdentityProviderCredentials,
30
- OAUTH_CONNECTION_TYPES
30
+ OAUTH_CONNECTION_TYPES,
31
+ ConnectionCredentialsEntryLocation,
32
+ ConnectionCredentialsEntry,
33
+ ConnectionCredentialsCustomFields
31
34
 
32
35
  )
33
36
 
@@ -167,6 +170,13 @@ def _validate_connection_params(type: ConnectionType, **args) -> None:
167
170
  f"Missing flags --grant-type is required for type {type}"
168
171
  )
169
172
 
173
+ if type != ConnectionType.OAUTH2_AUTH_CODE and (
174
+ args.get('auth_entries')
175
+ ):
176
+ raise typer.BadParameter(
177
+ f"The flag --auth-entries is only supported by type {type}"
178
+ )
179
+
170
180
 
171
181
  def _parse_entry(entry: str) -> dict[str,str]:
172
182
  split_entry = entry.split('=', 1)
@@ -176,6 +186,19 @@ def _parse_entry(entry: str) -> dict[str,str]:
176
186
  exit(1)
177
187
  return {split_entry[0]: split_entry[1]}
178
188
 
189
+ def _get_oauth_custom_fields(token_entries: List[ConnectionCredentialsEntry] | None, auth_entries: List[ConnectionCredentialsEntry] | None) -> dict:
190
+ custom_fields = ConnectionCredentialsCustomFields()
191
+
192
+ if token_entries:
193
+ for entry in token_entries:
194
+ custom_fields.add_field(entry, is_token=True)
195
+
196
+ if auth_entries:
197
+ for entry in auth_entries:
198
+ custom_fields.add_field(entry, is_token=False)
199
+
200
+ return custom_fields.model_dump(exclude_none=True)
201
+
179
202
  def _get_credentials(type: ConnectionType, **kwargs):
180
203
  match type:
181
204
  case ConnectionType.BASIC_AUTH:
@@ -192,18 +215,21 @@ def _get_credentials(type: ConnectionType, **kwargs):
192
215
  api_key=kwargs.get("api_key")
193
216
  )
194
217
  case ConnectionType.OAUTH2_AUTH_CODE:
218
+ custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
195
219
  return OAuth2AuthCodeCredentials(
196
220
  authorization_url=kwargs.get("auth_url"),
197
221
  client_id=kwargs.get("client_id"),
198
222
  client_secret=kwargs.get("client_secret"),
199
223
  token_url=kwargs.get("token_url"),
200
- scope=kwargs.get("scope")
224
+ scope=kwargs.get("scope"),
225
+ **custom_fields
201
226
  )
202
227
  case ConnectionType.OAUTH2_CLIENT_CREDS:
203
228
  # using filtered args as default values will not be set if 'None' is passed, causing validation errors
204
229
  keys = ["client_id","client_secret","token_url","grant_type","send_via", "scope"]
205
230
  filtered_args = { key_name: kwargs[key_name] for key_name in keys if kwargs.get(key_name) }
206
- return OAuth2ClientCredentials(**filtered_args)
231
+ custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
232
+ return OAuth2ClientCredentials(**filtered_args, **custom_fields)
207
233
  # case ConnectionType.OAUTH2_IMPLICIT:
208
234
  # return OAuth2ImplicitCredentials(
209
235
  # authorization_url=kwargs.get("auth_url"),
@@ -212,13 +238,16 @@ def _get_credentials(type: ConnectionType, **kwargs):
212
238
  case ConnectionType.OAUTH2_PASSWORD:
213
239
  keys = ["username", "password", "client_id","client_secret","token_url","grant_type", "scope"]
214
240
  filtered_args = { key_name: kwargs[key_name] for key_name in keys if kwargs.get(key_name) }
215
- return OAuth2PasswordCredentials(**filtered_args)
241
+ custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
242
+ return OAuth2PasswordCredentials(**filtered_args, **custom_fields)
216
243
 
217
244
  case ConnectionType.OAUTH_ON_BEHALF_OF_FLOW:
245
+ custom_fields = _get_oauth_custom_fields(kwargs.get("token_entries"), kwargs.get("auth_entries"))
218
246
  return OAuthOnBehalfOfCredentials(
219
247
  client_id=kwargs.get("client_id"),
220
248
  access_token_url=kwargs.get("token_url"),
221
- grant_type=kwargs.get("grant_type")
249
+ grant_type=kwargs.get("grant_type"),
250
+ **custom_fields
222
251
  )
223
252
  case ConnectionType.KEY_VALUE:
224
253
  env = {}
@@ -231,6 +260,23 @@ def _get_credentials(type: ConnectionType, **kwargs):
231
260
  case _:
232
261
  raise ValueError(f"Invalid type '{type}' selected")
233
262
 
263
+ def _connection_credentials_parse_entry(text: str, default_location: ConnectionCredentialsEntryLocation) -> ConnectionCredentialsEntry:
264
+ location_kv_pair = text.split(":", 1)
265
+ key_value = location_kv_pair[-1]
266
+ location = location_kv_pair[0] if len(location_kv_pair)>1 else default_location
267
+
268
+ valid_locations = [item.value for item in ConnectionCredentialsEntryLocation]
269
+ if location not in valid_locations:
270
+ raise typer.BadParameter(f"The provided location '{location}' is not in the allowed values {valid_locations}.")
271
+
272
+ key_value_pair = key_value.split('=', 1)
273
+ if len(key_value_pair) != 2:
274
+ message = f"The entry '{text}' is not in the expected form '<location>:<key>=<value>' or '<key>=<value>'"
275
+ raise typer.BadParameter(message)
276
+ key, value = key_value_pair[0], key_value_pair[1]
277
+
278
+ return ConnectionCredentialsEntry(key=key, value=value, location=location)
279
+
234
280
 
235
281
  def add_configuration(config: ConnectionConfiguration) -> None:
236
282
  client = get_connections_client()
@@ -524,5 +570,15 @@ def set_identity_provider_connection(
524
570
  logger.error(f"Cannot set Identity Provider when 'sso' is false in configuration. Please enable sso for connection '{app_id}' in environment '{environment}' and try again.")
525
571
  sys.exit(1)
526
572
 
527
- idp = IdentityProviderCredentials.model_validate(kwargs)
573
+ custom_fields = _get_oauth_custom_fields(token_entries=kwargs.get("token_entries"), auth_entries=None)
574
+ idp = IdentityProviderCredentials(**kwargs, **custom_fields)
528
575
  add_identity_provider(app_id=app_id, environment=environment, idp=idp)
576
+
577
+ def token_entry_connection_credentials_parse(text: str) -> ConnectionCredentialsEntry:
578
+ return _connection_credentials_parse_entry(text=text, default_location=ConnectionCredentialsEntryLocation.HEADER)
579
+
580
+ def auth_entry_connection_credentials_parse(text: str) -> ConnectionCredentialsEntry:
581
+ entry = _connection_credentials_parse_entry(text=text, default_location=ConnectionCredentialsEntryLocation.QUERY)
582
+ if entry.location != ConnectionCredentialsEntryLocation.QUERY:
583
+ raise typer.BadParameter(f"Only location '{ConnectionCredentialsEntryLocation.QUERY}' is supported for --auth-entry")
584
+ return entry
@@ -327,7 +327,11 @@ def talk_to_cpe(cpe_client, samples_file=None, context_data=None):
327
327
 
328
328
 
329
329
  def prompt_tune(agent_spec: str, output_file: str | None, samples_file: str | None, dry_run_flag: bool) -> None:
330
- agent = AgentsController.import_agent(file=agent_spec, app_id=None)[0]
330
+ agents = AgentsController.import_agent(file=agent_spec, app_id=None)
331
+ if not agents:
332
+ logger.error("Invalid agent spec file provided, no agent found.")
333
+ sys.exit(1)
334
+ agent = agents[0]
331
335
  agent_kind = agent.kind
332
336
 
333
337
  if agent_kind != AgentKind.NATIVE:
@@ -342,7 +346,7 @@ def prompt_tune(agent_spec: str, output_file: str | None, samples_file: str | No
342
346
 
343
347
  client = get_cpe_client()
344
348
 
345
- instr = agent.instructions
349
+ instr = agent.instructions if agent.instructions else ""
346
350
 
347
351
  tools = _get_tools_from_names(agent.tools)
348
352
 
@@ -70,7 +70,7 @@ def add_env(
70
70
  ] = None,
71
71
  type: Annotated[
72
72
  EnvironmentAuthType,
73
- typer.Option("--type", "-t", help="The type of auth you wish to use"),
73
+ typer.Option("--type", "-t", help="The type of auth you wish to use. This overrides the auth type that is inferred from the url"),
74
74
  ] = None,
75
75
  insecure: Annotated[
76
76
  bool,
@@ -16,7 +16,7 @@ from typing import Optional
16
16
  from typing_extensions import Annotated
17
17
 
18
18
  from ibm_watsonx_orchestrate import __version__
19
- from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController
19
+ from ibm_watsonx_orchestrate.cli.commands.evaluations.evaluations_controller import EvaluationsController, EvaluateMode
20
20
  from ibm_watsonx_orchestrate.cli.commands.agents.agents_controller import AgentsController
21
21
 
22
22
  logger = logging.getLogger(__name__)
@@ -220,6 +220,13 @@ def analyze(data_path: Annotated[
220
220
  help="Path to the directory that has the saved results"
221
221
  )
222
222
  ],
223
+ tool_definition_path: Annotated[
224
+ Optional[str],
225
+ typer.Option(
226
+ "--tools-path", "-t",
227
+ help="Path to the directory containing tool definitions."
228
+ )
229
+ ] = None,
223
230
  user_env_file: Annotated[
224
231
  Optional[str],
225
232
  typer.Option(
@@ -230,7 +237,10 @@ def analyze(data_path: Annotated[
230
237
 
231
238
  validate_watsonx_credentials(user_env_file)
232
239
  controller = EvaluationsController()
233
- controller.analyze(data_path=data_path)
240
+ controller.analyze(
241
+ data_path=data_path,
242
+ tool_definition_path=tool_definition_path
243
+ )
234
244
 
235
245
  @evaluation_app.command(name="validate-external", help="Validate an external agent against a set of inputs")
236
246
  def validate_external(
@@ -375,3 +385,165 @@ def validate_external(
375
385
  msg = f"[dark_orange]Schema validation did not succeed. See '{str(validation_folder)}' for failures.[/dark_orange]"
376
386
 
377
387
  rich.print(Panel(msg))
388
+
389
+ @evaluation_app.command(name="quick-eval",
390
+ short_help="Evaluate agent against a suite of static metrics and LLM-as-a-judge metrics",
391
+ help="""
392
+ Use the quick-eval command to evaluate your agent against a suite of static metrics and LLM-as-a-judge metrics.
393
+ """)
394
+ def quick_eval(
395
+ config_file: Annotated[
396
+ Optional[str],
397
+ typer.Option(
398
+ "--config", "-c",
399
+ help="Path to YAML configuration file containing evaluation settings."
400
+ )
401
+ ] = None,
402
+ test_paths: Annotated[
403
+ Optional[str],
404
+ typer.Option(
405
+ "--test-paths", "-p",
406
+ help="Paths to the test files and/or directories to evaluate, separated by commas."
407
+ ),
408
+ ] = None,
409
+ tools_path: Annotated[
410
+ str,
411
+ typer.Option(
412
+ "--tools-path", "-t",
413
+ help="Path to the directory containing tool definitions."
414
+ )
415
+ ] = None,
416
+ output_dir: Annotated[
417
+ Optional[str],
418
+ typer.Option(
419
+ "--output-dir", "-o",
420
+ help="Directory to save the evaluation results."
421
+ )
422
+ ] = None,
423
+ user_env_file: Annotated[
424
+ Optional[str],
425
+ typer.Option(
426
+ "--env-file", "-e",
427
+ help="Path to a .env file that overrides default.env. Then environment variables override both."
428
+ ),
429
+ ] = None
430
+ ):
431
+ if not config_file:
432
+ if not test_paths or not output_dir:
433
+ logger.error("Error: Both --test-paths and --output-dir must be provided when not using a config file")
434
+ exit(1)
435
+
436
+ validate_watsonx_credentials(user_env_file)
437
+
438
+ if tools_path is None:
439
+ logger.error("When running `quick-eval`, please provide the path to your tools file.")
440
+ sys.exit(1)
441
+
442
+ controller = EvaluationsController()
443
+ controller.evaluate(
444
+ config_file=config_file,
445
+ test_paths=test_paths,
446
+ output_dir=output_dir,
447
+ tools_path=tools_path, mode=EvaluateMode.referenceless
448
+ )
449
+
450
+
451
+ red_teaming_app = typer.Typer(no_args_is_help=True)
452
+ evaluation_app.add_typer(red_teaming_app, name="red-teaming")
453
+
454
+
455
+ @red_teaming_app.command("list", help="List available red-teaming attack plans")
456
+ def list_plans():
457
+ controller = EvaluationsController()
458
+ controller.list_red_teaming_attacks()
459
+
460
+
461
+ @red_teaming_app.command("plan", help="Generate red-teaming attacks")
462
+ def plan(
463
+ attacks_list: Annotated[
464
+ str,
465
+ typer.Option(
466
+ "--attacks-list",
467
+ "-a",
468
+ help="Comma-separated list of red-teaming attacks to generate.",
469
+ ),
470
+ ],
471
+ datasets_path: Annotated[
472
+ str,
473
+ typer.Option(
474
+ "--datasets-path",
475
+ "-d",
476
+ help="Path to datasets for red-teaming. This can also be a comma-separated list of files or directories.",
477
+ ),
478
+ ],
479
+ agents_path: Annotated[
480
+ str, typer.Option("--agents-path", "-g", help="Path to the directory containing all agent definitions.")
481
+ ],
482
+ target_agent_name: Annotated[
483
+ str,
484
+ typer.Option(
485
+ "--target-agent-name",
486
+ "-t",
487
+ help="Name of the target agent to attack (should be present in agents-path).",
488
+ ),
489
+ ],
490
+ output_dir: Annotated[
491
+ Optional[str],
492
+ typer.Option("--output-dir", "-o", help="Directory to save generated attacks.")
493
+ ]=None,
494
+ user_env_file: Annotated[
495
+ Optional[str],
496
+ typer.Option(
497
+ "--env-file",
498
+ "-e",
499
+ help="Path to a .env file that overrides default.env. Then environment variables override both.",
500
+ ),
501
+ ] = None,
502
+ max_variants: Annotated[
503
+ Optional[int],
504
+ typer.Option(
505
+ "--max_variants",
506
+ "-n",
507
+ help="Number of variants to generate per attack type.",
508
+ ),
509
+ ] = None,
510
+
511
+ ):
512
+ validate_watsonx_credentials(user_env_file)
513
+ controller = EvaluationsController()
514
+ controller.generate_red_teaming_attacks(
515
+ attacks_list=attacks_list,
516
+ datasets_path=datasets_path,
517
+ agents_path=agents_path,
518
+ target_agent_name=target_agent_name,
519
+ output_dir=output_dir,
520
+ max_variants=max_variants,
521
+ )
522
+
523
+
524
+ @red_teaming_app.command("run", help="Run red-teaming attacks")
525
+ def run(
526
+ attack_paths: Annotated[
527
+ str,
528
+ typer.Option(
529
+ "--attack-paths",
530
+ "-a",
531
+ help="Path or list of paths (comma-separated) to directories containing attack files.",
532
+ ),
533
+ ],
534
+ output_dir: Annotated[
535
+ Optional[str],
536
+ typer.Option("--output-dir", "-o", help="Directory to save attack results."),
537
+ ] = None,
538
+ user_env_file: Annotated[
539
+ Optional[str],
540
+ typer.Option(
541
+ "--env-file",
542
+ "-e",
543
+ help="Path to a .env file that overrides default.env. Then environment variables override both.",
544
+ ),
545
+ ] = None,
546
+ ):
547
+ validate_watsonx_credentials(user_env_file)
548
+ controller = EvaluationsController()
549
+ controller.run_red_teaming_attacks(attack_paths=attack_paths, output_dir=output_dir)
@@ -1,17 +1,23 @@
1
1
  import logging
2
2
  import os.path
3
3
  from typing import List, Dict, Optional, Tuple
4
+ from enum import StrEnum
4
5
  import csv
5
6
  from pathlib import Path
6
7
  import sys
7
8
  from wxo_agentic_evaluation import main as evaluate
9
+ from wxo_agentic_evaluation import quick_eval
8
10
  from wxo_agentic_evaluation.tool_planner import build_snapshot
9
- from wxo_agentic_evaluation.analyze_run import analyze
11
+ from wxo_agentic_evaluation.analyze_run import Analyzer
10
12
  from wxo_agentic_evaluation.batch_annotate import generate_test_cases_from_stories
11
- from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig, ProviderConfig
13
+ from wxo_agentic_evaluation.arg_configs import TestConfig, AuthConfig, LLMUserConfig, ChatRecordingConfig, AnalyzeConfig, ProviderConfig, AttackConfig, QuickEvalConfig
12
14
  from wxo_agentic_evaluation.record_chat import record_chats
13
15
  from wxo_agentic_evaluation.external_agent.external_validate import ExternalAgentValidation
14
16
  from wxo_agentic_evaluation.external_agent.performance_test import ExternalAgentPerformanceTest
17
+ from wxo_agentic_evaluation.red_teaming.attack_list import print_attacks
18
+ from wxo_agentic_evaluation.red_teaming import attack_generator
19
+ from wxo_agentic_evaluation.red_teaming.attack_runner import run_attacks
20
+ from wxo_agentic_evaluation.arg_configs import AttackGeneratorConfig
15
21
  from ibm_watsonx_orchestrate import __version__
16
22
  from ibm_watsonx_orchestrate.cli.config import Config, ENV_WXO_URL_OPT, AUTH_CONFIG_FILE, AUTH_CONFIG_FILE_FOLDER, AUTH_SECTION_HEADER, AUTH_MCSP_TOKEN_OPT
17
23
  from ibm_watsonx_orchestrate.utils.utils import yaml_safe_load
@@ -21,6 +27,9 @@ import uuid
21
27
 
22
28
  logger = logging.getLogger(__name__)
23
29
 
30
+ class EvaluateMode(StrEnum):
31
+ default = "default" # referenceFUL evaluation
32
+ referenceless = "referenceless"
24
33
 
25
34
  class EvaluationsController:
26
35
  def __init__(self):
@@ -38,7 +47,7 @@ class EvaluationsController:
38
47
 
39
48
  return url, tenant_name, token
40
49
 
41
- def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None) -> None:
50
+ def evaluate(self, config_file: Optional[str] = None, test_paths: Optional[str] = None, output_dir: Optional[str] = None, tools_path: str = None, mode: str = EvaluateMode.default) -> None:
42
51
  url, tenant_name, token = self._get_env_config()
43
52
 
44
53
  if "WATSONX_SPACE_ID" in os.environ and "WATSONX_APIKEY" in os.environ:
@@ -90,9 +99,13 @@ class EvaluationsController:
90
99
  config_data["output_dir"] = output_dir
91
100
  logger.info(f"Using output directory: {config_data['output_dir']}")
92
101
 
93
- config = TestConfig(**config_data)
94
-
95
- evaluate.main(config)
102
+ if mode == EvaluateMode.default:
103
+ config = TestConfig(**config_data)
104
+ evaluate.main(config)
105
+ elif mode == EvaluateMode.referenceless:
106
+ config_data["tools_path"] = tools_path
107
+ config = QuickEvalConfig(**config_data)
108
+ quick_eval.main(config)
96
109
 
97
110
  def record(self, output_dir) -> None:
98
111
 
@@ -160,9 +173,13 @@ class EvaluationsController:
160
173
 
161
174
  logger.info("Test cases stored at: %s", output_dir)
162
175
 
163
- def analyze(self, data_path: str) -> None:
164
- config = AnalyzeConfig(data_path=data_path)
165
- analyze(config)
176
+ def analyze(self, data_path: str, tool_definition_path: str) -> None:
177
+ config = AnalyzeConfig(
178
+ data_path=data_path,
179
+ tool_definition_path=tool_definition_path
180
+ )
181
+ analyzer = Analyzer()
182
+ analyzer.analyze(config)
166
183
 
167
184
  def summarize(self) -> None:
168
185
  pass
@@ -187,3 +204,70 @@ class EvaluationsController:
187
204
  generated_performance_tests = performance_test.generate_tests()
188
205
 
189
206
  return generated_performance_tests
207
+
208
+ def list_red_teaming_attacks(self):
209
+ print_attacks()
210
+
211
+ def generate_red_teaming_attacks(
212
+ self,
213
+ attacks_list: str,
214
+ datasets_path: str,
215
+ agents_path: str,
216
+ target_agent_name: str,
217
+ output_dir: Optional[str] = None,
218
+ max_variants: Optional[int] = None,
219
+ ):
220
+ if output_dir is None:
221
+ output_dir = os.path.join(os.getcwd(), "red_teaming_attacks")
222
+ os.makedirs(output_dir, exist_ok=True)
223
+ logger.info(f"No output directory specified. Using default: {output_dir}")
224
+
225
+ results = attack_generator.main(
226
+ AttackGeneratorConfig(
227
+ attacks_list=attacks_list.split(","),
228
+ datasets_path=datasets_path.split(","),
229
+ agents_path=agents_path,
230
+ target_agent_name=target_agent_name,
231
+ output_dir=output_dir,
232
+ max_variants=max_variants,
233
+ )
234
+ )
235
+ logger.info(f"Generated {len(results)} attacks and saved to {output_dir}")
236
+
237
+ def run_red_teaming_attacks(self, attack_paths: str, output_dir: Optional[str] = None) -> None:
238
+ url, tenant_name, token = self._get_env_config()
239
+
240
+ if "WATSONX_SPACE_ID" in os.environ and "WATSONX_APIKEY" in os.environ:
241
+ provider = "watsonx"
242
+ elif "WO_INSTANCE" in os.environ and "WO_API_KEY" in os.environ:
243
+ provider = "model_proxy"
244
+ else:
245
+ logger.error(
246
+ "No provider found. Please either provide a config_file or set either WATSONX_SPACE_ID and WATSONX_APIKEY or WO_INSTANCE and WO_API_KEY in your system environment variables."
247
+ )
248
+ sys.exit(1)
249
+
250
+ config_data = {
251
+ "auth_config": AuthConfig(
252
+ url=url,
253
+ tenant_name=tenant_name,
254
+ token=token,
255
+ ),
256
+ "provider_config": ProviderConfig(
257
+ provider=provider,
258
+ model_id="meta-llama/llama-3-405b-instruct",
259
+ ),
260
+ }
261
+
262
+ config_data["attack_paths"] = attack_paths.split(",")
263
+ if output_dir:
264
+ config_data["output_dir"] = output_dir
265
+ else:
266
+ config_data["output_dir"] = os.path.join(os.getcwd(), "red_teaming_results")
267
+ os.makedirs(config_data["output_dir"], exist_ok=True)
268
+ logger.info(f"No output directory specified. Using default: {config_data['output_dir']}")
269
+
270
+
271
+ config = AttackConfig(**config_data)
272
+
273
+ run_attacks(config)
@@ -191,9 +191,6 @@ def get_default_registry_env_vars_by_dev_edition_source(default_env: dict, user_
191
191
  parsed = urlparse(wo_url)
192
192
  hostname = parsed.hostname
193
193
 
194
- if not hostname or not hostname.startswith("api."):
195
- raise ValueError(f"Invalid WO_INSTANCE URL: '{wo_url}'. It should starts with 'api.'")
196
-
197
194
  registry_url = f"registry.{hostname[4:]}/cp/wxo-lite"
198
195
  else:
199
196
  raise ValueError(f"Unknown value for developer edition source: {source}. Must be one of ['internal', 'myibm', 'orchestrate'].")
@@ -1,5 +1,6 @@
1
1
  import logging
2
2
  import sys
3
+ import uuid
3
4
  from enum import Enum
4
5
  from pydantic import BaseModel, model_validator, ConfigDict
5
6
 
@@ -43,9 +44,6 @@ class WatsonXAIEnvConfig(BaseModel):
43
44
  if not config.get("WATSONX_SPACE_ID") and not config.get("WATSONX_APIKEY"):
44
45
  raise ValueError("Missing configuration requirements 'WATSONX_SPACE_ID' and 'WATSONX_APIKEY'")
45
46
 
46
- if config.get("WATSONX_SPACE_ID") and not config.get("WATSONX_APIKEY"):
47
- logger.error("Cannot use env var 'WATSONX_SPACE_ID' without setting the corresponding 'WATSONX_APIKEY'")
48
- sys.exit(1)
49
47
 
50
48
  if not config.get("WATSONX_SPACE_ID") and config.get("WATSONX_APIKEY"):
51
49
  logger.error("Cannot use env var 'WATSONX_APIKEY' without setting the corresponding 'WATSONX_SPACE_ID'")
@@ -54,6 +52,12 @@ class WatsonXAIEnvConfig(BaseModel):
54
52
  config["USE_SAAS_ML_TOOLS_RUNTIME"] = False
55
53
  return config
56
54
 
55
+ def is_valid_uuid(value) -> bool:
56
+ try:
57
+ uuid.UUID(str(value))
58
+ return True
59
+ except (ValueError, TypeError, AttributeError):
60
+ return False
57
61
 
58
62
  class ModelGatewayEnvConfig(BaseModel):
59
63
  WO_API_KEY: str | None = None
@@ -84,8 +88,11 @@ class ModelGatewayEnvConfig(BaseModel):
84
88
  if not config.get("AUTHORIZATION_URL"):
85
89
  inferred_auth_url = AUTH_TYPE_DEFAULT_URL_MAPPING.get(auth_type)
86
90
  if not inferred_auth_url:
87
- logger.error(f"No 'AUTHORIZATION_URL' found. Auth type '{auth_type}' does not support defaulting. Please set the 'AUTHORIZATION_URL' explictly")
88
- sys.exit(1)
91
+ if auth_type == WoAuthType.CPD:
92
+ inferred_auth_url = config.get("WO_INSTANCE") + '/icp4d-api/v1/authorize'
93
+ else:
94
+ logger.error(f"No 'AUTHORIZATION_URL' found. Auth type '{auth_type}' does not support defaulting. Please set the 'AUTHORIZATION_URL' explictly")
95
+ sys.exit(1)
89
96
  config["AUTHORIZATION_URL"] = inferred_auth_url
90
97
 
91
98
  if auth_type != WoAuthType.CPD:
@@ -101,6 +108,7 @@ class ModelGatewayEnvConfig(BaseModel):
101
108
  sys.exit(1)
102
109
 
103
110
  config["USE_SAAS_ML_TOOLS_RUNTIME"] = True
104
- # Fake (but valid) UUIDv4 for knowledgebase check
105
- config["WATSONX_SPACE_ID"] = "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa"
111
+ if not is_valid_uuid(config.get("WATSONX_SPACE_ID")):
112
+ # Fake (but valid) UUIDv4 for knowledgebase check
113
+ config["WATSONX_SPACE_ID"] = "aaaaaaaa-aaaa-4aaa-aaaa-aaaaaaaaaaaa"
106
114
  return config