enkryptai-sdk 1.0.24__py3-none-any.whl → 1.0.26__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- enkryptai_sdk/__init__.py +63 -5
- enkryptai_sdk/dto/__init__.py +9 -0
- enkryptai_sdk/dto/datasets.py +66 -16
- enkryptai_sdk/dto/models.py +81 -17
- enkryptai_sdk/dto/red_team.py +396 -82
- enkryptai_sdk/red_team.py +270 -58
- enkryptai_sdk/response.py +271 -0
- enkryptai_sdk/utils/__init__.py +29 -0
- enkryptai_sdk/utils/pagination.py +384 -0
- {enkryptai_sdk-1.0.24.dist-info → enkryptai_sdk-1.0.26.dist-info}/METADATA +1 -1
- {enkryptai_sdk-1.0.24.dist-info → enkryptai_sdk-1.0.26.dist-info}/RECORD +14 -12
- {enkryptai_sdk-1.0.24.dist-info → enkryptai_sdk-1.0.26.dist-info}/WHEEL +0 -0
- {enkryptai_sdk-1.0.24.dist-info → enkryptai_sdk-1.0.26.dist-info}/licenses/LICENSE +0 -0
- {enkryptai_sdk-1.0.24.dist-info → enkryptai_sdk-1.0.26.dist-info}/top_level.txt +0 -0
enkryptai_sdk/dto/red_team.py
CHANGED
|
@@ -9,6 +9,7 @@ from .models import ModelConfig
|
|
|
9
9
|
from .guardrails import GuardrailDetectors
|
|
10
10
|
from .common import ModelAuthTypeEnum, CustomHeader, ModelJwtConfig
|
|
11
11
|
|
|
12
|
+
|
|
12
13
|
# The risk mitigation do not support all detectors, so we need to create a separate enum for them.
|
|
13
14
|
class RiskGuardrailDetectorsEnum(str, Enum):
|
|
14
15
|
NSFW = "nsfw"
|
|
@@ -27,17 +28,13 @@ class RiskGuardrailDetectorsEnum(str, Enum):
|
|
|
27
28
|
class RedteamHealthResponse(BaseDTO):
|
|
28
29
|
status: str
|
|
29
30
|
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
30
|
-
|
|
31
|
+
|
|
31
32
|
@classmethod
|
|
32
33
|
def from_dict(cls, data: Dict[str, Any]) -> "RedteamHealthResponse":
|
|
33
|
-
return cls(
|
|
34
|
-
|
|
35
|
-
)
|
|
36
|
-
|
|
34
|
+
return cls(status=data.get("status", ""))
|
|
35
|
+
|
|
37
36
|
def to_dict(self) -> Dict[str, Any]:
|
|
38
|
-
return {
|
|
39
|
-
"status": self.status
|
|
40
|
-
}
|
|
37
|
+
return {"status": self.status}
|
|
41
38
|
|
|
42
39
|
|
|
43
40
|
@dataclass
|
|
@@ -55,7 +52,7 @@ class RedTeamResponse(BaseDTO):
|
|
|
55
52
|
message=data.get("message"),
|
|
56
53
|
data=data.get("data"),
|
|
57
54
|
)
|
|
58
|
-
|
|
55
|
+
|
|
59
56
|
def to_dict(self) -> Dict:
|
|
60
57
|
return super().to_dict()
|
|
61
58
|
|
|
@@ -77,7 +74,7 @@ class RedTeamTaskDetailsModelConfig(BaseDTO):
|
|
|
77
74
|
system_prompt=data.get("system_prompt"),
|
|
78
75
|
model_version=data.get("model_version"),
|
|
79
76
|
)
|
|
80
|
-
|
|
77
|
+
|
|
81
78
|
def to_dict(self) -> Dict:
|
|
82
79
|
return {
|
|
83
80
|
"system_prompt": self.system_prompt,
|
|
@@ -108,9 +105,11 @@ class RedTeamTaskDetails(BaseDTO):
|
|
|
108
105
|
status=data.get("status"),
|
|
109
106
|
test_name=data.get("test_name"),
|
|
110
107
|
task_id=data.get("task_id"),
|
|
111
|
-
model_config=RedTeamTaskDetailsModelConfig.from_dict(
|
|
108
|
+
model_config=RedTeamTaskDetailsModelConfig.from_dict(
|
|
109
|
+
data.get("model_config", {})
|
|
110
|
+
),
|
|
112
111
|
)
|
|
113
|
-
|
|
112
|
+
|
|
114
113
|
def to_dict(self) -> Dict:
|
|
115
114
|
return {
|
|
116
115
|
"created_at": self.created_at,
|
|
@@ -191,8 +190,10 @@ class ResultSummary(BaseDTO):
|
|
|
191
190
|
for key, value in item.items():
|
|
192
191
|
result[key] = StatisticItem.from_dict(value)
|
|
193
192
|
return result
|
|
194
|
-
|
|
195
|
-
def convert_stat_test_type_list(
|
|
193
|
+
|
|
194
|
+
def convert_stat_test_type_list(
|
|
195
|
+
stat_list: List[Dict],
|
|
196
|
+
) -> Dict[str, StatisticItemWithTestType]:
|
|
196
197
|
result = {}
|
|
197
198
|
for item in stat_list:
|
|
198
199
|
for key, value in item.items():
|
|
@@ -221,8 +222,10 @@ class ResultSummary(BaseDTO):
|
|
|
221
222
|
def to_dict(self) -> Dict:
|
|
222
223
|
def convert_stat_dict(stat_dict: Dict[str, StatisticItem]) -> List[Dict]:
|
|
223
224
|
return [{key: value.to_dict()} for key, value in stat_dict.items()]
|
|
224
|
-
|
|
225
|
-
def convert_stat_test_type_dict(
|
|
225
|
+
|
|
226
|
+
def convert_stat_test_type_dict(
|
|
227
|
+
stat_dict: Dict[str, StatisticItemWithTestType],
|
|
228
|
+
) -> List[Dict]:
|
|
226
229
|
return [{key: value.to_dict()} for key, value in stat_dict.items()]
|
|
227
230
|
|
|
228
231
|
d = super().to_dict()
|
|
@@ -248,9 +251,11 @@ class RedTeamResultSummary(BaseDTO):
|
|
|
248
251
|
def from_dict(cls, data: Dict) -> "RedTeamResultSummary":
|
|
249
252
|
if not data or "summary" not in data:
|
|
250
253
|
return cls(summary=ResultSummary.from_dict({}))
|
|
251
|
-
|
|
254
|
+
|
|
252
255
|
if "task_status" in data:
|
|
253
|
-
return cls(
|
|
256
|
+
return cls(
|
|
257
|
+
summary=ResultSummary.from_dict({}), task_status=data["task_status"]
|
|
258
|
+
)
|
|
254
259
|
|
|
255
260
|
return cls(summary=ResultSummary.from_dict(data["summary"]))
|
|
256
261
|
|
|
@@ -295,22 +300,22 @@ class RedTeamResultDetails(BaseDTO):
|
|
|
295
300
|
def from_dict(cls, data: Dict) -> "RedTeamResultDetails":
|
|
296
301
|
if not data or "details" not in data:
|
|
297
302
|
return cls(details=[])
|
|
298
|
-
|
|
303
|
+
|
|
299
304
|
if "task_status" in data:
|
|
300
305
|
return cls(details=[], task_status=data["task_status"])
|
|
301
306
|
|
|
302
307
|
# details = []
|
|
303
308
|
# for result in data["details"]:
|
|
304
|
-
|
|
305
|
-
|
|
309
|
+
# Convert eval_tokens dict to TestEvalTokens object
|
|
310
|
+
# eval_tokens = TestEvalTokens(**result["eval_tokens"])
|
|
306
311
|
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
312
|
+
# Create a copy of the result dict and replace eval_tokens
|
|
313
|
+
# result_copy = dict(result["details"])
|
|
314
|
+
# result_copy["eval_tokens"] = eval_tokens
|
|
310
315
|
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
316
|
+
# Create TestResult object
|
|
317
|
+
# test_result = TestResult(**result_copy)
|
|
318
|
+
# details.append(test_result)
|
|
314
319
|
|
|
315
320
|
return cls(details=data["details"])
|
|
316
321
|
|
|
@@ -334,10 +339,23 @@ class RedTeamResultDetails(BaseDTO):
|
|
|
334
339
|
@dataclass
|
|
335
340
|
class AttackMethods(BaseDTO):
|
|
336
341
|
basic: List[str] = field(default_factory=lambda: ["basic"])
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
342
|
+
advanced: Dict[str, List[str]] = field(
|
|
343
|
+
default_factory=lambda: {
|
|
344
|
+
"static": [
|
|
345
|
+
"masking",
|
|
346
|
+
"figstep",
|
|
347
|
+
"hades",
|
|
348
|
+
"encoding",
|
|
349
|
+
"single_shot",
|
|
350
|
+
"echo",
|
|
351
|
+
"speed",
|
|
352
|
+
"pitch",
|
|
353
|
+
"reverb",
|
|
354
|
+
"noise",
|
|
355
|
+
],
|
|
356
|
+
"dynamic": ["iterative", "jood"],
|
|
357
|
+
}
|
|
358
|
+
)
|
|
341
359
|
|
|
342
360
|
def to_dict(self) -> dict:
|
|
343
361
|
return asdict(self)
|
|
@@ -448,7 +466,7 @@ class OutputModality(str, Enum):
|
|
|
448
466
|
# audio = "audio"
|
|
449
467
|
# video = "video"
|
|
450
468
|
# code = "code"
|
|
451
|
-
|
|
469
|
+
|
|
452
470
|
|
|
453
471
|
@dataclass
|
|
454
472
|
class TargetModelConfiguration(BaseDTO):
|
|
@@ -476,13 +494,17 @@ class TargetModelConfiguration(BaseDTO):
|
|
|
476
494
|
def from_dict(cls, data: dict):
|
|
477
495
|
data = data.copy()
|
|
478
496
|
if "custom_headers" in data:
|
|
479
|
-
data["custom_headers"] = [
|
|
497
|
+
data["custom_headers"] = [
|
|
498
|
+
CustomHeader.from_dict(header) for header in data["custom_headers"]
|
|
499
|
+
]
|
|
480
500
|
if "model_auth_type" in data:
|
|
481
501
|
data["model_auth_type"] = ModelAuthTypeEnum(data["model_auth_type"])
|
|
482
502
|
if "model_jwt_config" in data:
|
|
483
|
-
data["model_jwt_config"] = ModelJwtConfig.from_dict(
|
|
503
|
+
data["model_jwt_config"] = ModelJwtConfig.from_dict(
|
|
504
|
+
data["model_jwt_config"]
|
|
505
|
+
)
|
|
484
506
|
return cls(**data)
|
|
485
|
-
|
|
507
|
+
|
|
486
508
|
def to_dict(self) -> dict:
|
|
487
509
|
d = asdict(self)
|
|
488
510
|
d["model_auth_type"] = self.model_auth_type.value
|
|
@@ -515,6 +537,67 @@ class RedTeamModelHealthConfig(BaseDTO):
|
|
|
515
537
|
)
|
|
516
538
|
|
|
517
539
|
|
|
540
|
+
@dataclass
|
|
541
|
+
class RedTeamModelHealthConfigV3(BaseDTO):
|
|
542
|
+
"""
|
|
543
|
+
V3 format for model health check that accepts endpoint_configuration
|
|
544
|
+
similar to add_custom_task.
|
|
545
|
+
"""
|
|
546
|
+
|
|
547
|
+
endpoint_configuration: ModelConfig = field(default_factory=ModelConfig)
|
|
548
|
+
|
|
549
|
+
def to_dict(self) -> dict:
|
|
550
|
+
d = asdict(self)
|
|
551
|
+
d["endpoint_configuration"] = self.endpoint_configuration.to_dict()
|
|
552
|
+
return d
|
|
553
|
+
|
|
554
|
+
@classmethod
|
|
555
|
+
def from_dict(cls, data: dict):
|
|
556
|
+
data = data.copy()
|
|
557
|
+
endpoint_config = ModelConfig.from_dict(data.pop("endpoint_configuration", {}))
|
|
558
|
+
return cls(
|
|
559
|
+
endpoint_configuration=endpoint_config,
|
|
560
|
+
)
|
|
561
|
+
|
|
562
|
+
def to_target_model_configuration(self) -> TargetModelConfiguration:
|
|
563
|
+
"""
|
|
564
|
+
Convert endpoint_configuration to target_model_configuration format.
|
|
565
|
+
This enables the V3 format to be compatible with the existing backend API.
|
|
566
|
+
"""
|
|
567
|
+
model_config = self.endpoint_configuration.model_config
|
|
568
|
+
|
|
569
|
+
return TargetModelConfiguration(
|
|
570
|
+
testing_for=self.endpoint_configuration.testing_for,
|
|
571
|
+
system_prompt=model_config.system_prompt,
|
|
572
|
+
model_source=model_config.model_source,
|
|
573
|
+
model_provider=(
|
|
574
|
+
model_config.model_provider.value
|
|
575
|
+
if hasattr(model_config.model_provider, "value")
|
|
576
|
+
else model_config.model_provider
|
|
577
|
+
),
|
|
578
|
+
model_endpoint_url=model_config.endpoint_url,
|
|
579
|
+
rate_per_min=model_config.rate_per_min,
|
|
580
|
+
model_name=self.endpoint_configuration.model_name,
|
|
581
|
+
model_version=self.endpoint_configuration.model_version,
|
|
582
|
+
model_auth_type=model_config.model_auth_type,
|
|
583
|
+
model_jwt_config=model_config.model_jwt_config,
|
|
584
|
+
model_api_key=model_config.apikey,
|
|
585
|
+
input_modalities=[
|
|
586
|
+
InputModality(m) if isinstance(m, str) else m
|
|
587
|
+
for m in model_config.input_modalities
|
|
588
|
+
],
|
|
589
|
+
output_modalities=[
|
|
590
|
+
OutputModality(m) if isinstance(m, str) else m
|
|
591
|
+
for m in model_config.output_modalities
|
|
592
|
+
],
|
|
593
|
+
custom_curl_command=model_config.custom_curl_command,
|
|
594
|
+
custom_headers=model_config.custom_headers,
|
|
595
|
+
custom_payload=model_config.custom_payload,
|
|
596
|
+
custom_response_content_type=model_config.custom_response_content_type,
|
|
597
|
+
custom_response_format=model_config.custom_response_format,
|
|
598
|
+
)
|
|
599
|
+
|
|
600
|
+
|
|
518
601
|
@dataclass
|
|
519
602
|
class RedteamModelHealthResponse(BaseDTO):
|
|
520
603
|
status: str
|
|
@@ -522,22 +605,22 @@ class RedteamModelHealthResponse(BaseDTO):
|
|
|
522
605
|
error: str
|
|
523
606
|
data: Optional[Dict[str, Any]] = field(default_factory=dict)
|
|
524
607
|
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
525
|
-
|
|
608
|
+
|
|
526
609
|
@classmethod
|
|
527
610
|
def from_dict(cls, data: Dict[str, Any]) -> "RedteamModelHealthResponse":
|
|
528
611
|
return cls(
|
|
529
612
|
status=data.get("status", ""),
|
|
530
613
|
message=data.get("message", ""),
|
|
531
614
|
data=data.get("data", {}),
|
|
532
|
-
error=data.get("error", "")
|
|
615
|
+
error=data.get("error", ""),
|
|
533
616
|
)
|
|
534
|
-
|
|
617
|
+
|
|
535
618
|
def to_dict(self) -> Dict[str, Any]:
|
|
536
619
|
return {
|
|
537
620
|
"status": self.status,
|
|
538
621
|
"message": self.message,
|
|
539
622
|
"data": self.data,
|
|
540
|
-
"error": self.error
|
|
623
|
+
"error": self.error,
|
|
541
624
|
}
|
|
542
625
|
|
|
543
626
|
|
|
@@ -610,24 +693,27 @@ class RedTeamConfigWithSavedModel(BaseDTO):
|
|
|
610
693
|
@dataclass
|
|
611
694
|
class RedTeamCustomConfig(BaseDTO):
|
|
612
695
|
test_name: str = "Test Name"
|
|
696
|
+
frameworks: List[str] = field(default_factory=list)
|
|
613
697
|
|
|
614
698
|
redteam_test_configurations: RedTeamTestConfigurations = field(
|
|
615
699
|
default_factory=RedTeamTestConfigurations
|
|
616
700
|
)
|
|
617
|
-
dataset_configuration: DatasetConfig =
|
|
618
|
-
|
|
619
|
-
)
|
|
620
|
-
endpoint_configuration: ModelConfig = field(
|
|
621
|
-
default_factory=ModelConfig
|
|
622
|
-
)
|
|
701
|
+
dataset_configuration: Optional[DatasetConfig] = None
|
|
702
|
+
endpoint_configuration: Optional[ModelConfig] = None
|
|
623
703
|
|
|
624
704
|
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
625
705
|
|
|
626
706
|
def to_dict(self) -> dict:
|
|
627
707
|
d = asdict(self)
|
|
628
708
|
d["redteam_test_configurations"] = self.redteam_test_configurations.to_dict()
|
|
629
|
-
|
|
630
|
-
|
|
709
|
+
if self.dataset_configuration is not None:
|
|
710
|
+
d["dataset_configuration"] = self.dataset_configuration.to_dict()
|
|
711
|
+
else:
|
|
712
|
+
d.pop("dataset_configuration", None)
|
|
713
|
+
if self.endpoint_configuration is not None:
|
|
714
|
+
d["endpoint_configuration"] = self.endpoint_configuration.to_dict()
|
|
715
|
+
else:
|
|
716
|
+
d.pop("endpoint_configuration", None)
|
|
631
717
|
return d
|
|
632
718
|
|
|
633
719
|
@classmethod
|
|
@@ -636,12 +722,18 @@ class RedTeamCustomConfig(BaseDTO):
|
|
|
636
722
|
test_configs = RedTeamTestConfigurations.from_dict(
|
|
637
723
|
data.pop("redteam_test_configurations", {})
|
|
638
724
|
)
|
|
639
|
-
dataset_config =
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
data.pop("
|
|
644
|
-
|
|
725
|
+
dataset_config = None
|
|
726
|
+
if "dataset_configuration" in data and data["dataset_configuration"]:
|
|
727
|
+
dataset_config = DatasetConfig.from_dict(data.pop("dataset_configuration"))
|
|
728
|
+
else:
|
|
729
|
+
data.pop("dataset_configuration", None)
|
|
730
|
+
|
|
731
|
+
endpoint_config = None
|
|
732
|
+
if "endpoint_configuration" in data and data["endpoint_configuration"]:
|
|
733
|
+
endpoint_config = ModelConfig.from_dict(data.pop("endpoint_configuration"))
|
|
734
|
+
else:
|
|
735
|
+
data.pop("endpoint_configuration", None)
|
|
736
|
+
|
|
645
737
|
return cls(
|
|
646
738
|
**data,
|
|
647
739
|
redteam_test_configurations=test_configs,
|
|
@@ -649,23 +741,26 @@ class RedTeamCustomConfig(BaseDTO):
|
|
|
649
741
|
endpoint_configuration=endpoint_config,
|
|
650
742
|
)
|
|
651
743
|
|
|
744
|
+
|
|
652
745
|
@dataclass
|
|
653
746
|
class RedTeamCustomConfigWithSavedModel(BaseDTO):
|
|
654
747
|
test_name: str = "Test Name"
|
|
748
|
+
frameworks: List[str] = field(default_factory=list)
|
|
655
749
|
|
|
656
750
|
redteam_test_configurations: RedTeamTestConfigurations = field(
|
|
657
751
|
default_factory=RedTeamTestConfigurations
|
|
658
752
|
)
|
|
659
|
-
dataset_configuration: DatasetConfig =
|
|
660
|
-
default_factory=DatasetConfig
|
|
661
|
-
)
|
|
753
|
+
dataset_configuration: Optional[DatasetConfig] = None
|
|
662
754
|
|
|
663
755
|
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
664
756
|
|
|
665
757
|
def to_dict(self) -> dict:
|
|
666
758
|
d = asdict(self)
|
|
667
759
|
d["redteam_test_configurations"] = self.redteam_test_configurations.to_dict()
|
|
668
|
-
|
|
760
|
+
if self.dataset_configuration is not None:
|
|
761
|
+
d["dataset_configuration"] = self.dataset_configuration.to_dict()
|
|
762
|
+
else:
|
|
763
|
+
d.pop("dataset_configuration", None)
|
|
669
764
|
return d
|
|
670
765
|
|
|
671
766
|
@classmethod
|
|
@@ -674,9 +769,12 @@ class RedTeamCustomConfigWithSavedModel(BaseDTO):
|
|
|
674
769
|
test_configs = RedTeamTestConfigurations.from_dict(
|
|
675
770
|
data.pop("redteam_test_configurations", {})
|
|
676
771
|
)
|
|
677
|
-
dataset_config =
|
|
678
|
-
|
|
679
|
-
|
|
772
|
+
dataset_config = None
|
|
773
|
+
if "dataset_configuration" in data and data["dataset_configuration"]:
|
|
774
|
+
dataset_config = DatasetConfig.from_dict(data.pop("dataset_configuration"))
|
|
775
|
+
else:
|
|
776
|
+
data.pop("dataset_configuration", None)
|
|
777
|
+
|
|
680
778
|
return cls(
|
|
681
779
|
**data,
|
|
682
780
|
redteam_test_configurations=test_configs,
|
|
@@ -691,7 +789,7 @@ class RedTeamTaskList(BaseDTO):
|
|
|
691
789
|
def to_dataframe(self) -> pd.DataFrame:
|
|
692
790
|
data = [task for task in self.tasks]
|
|
693
791
|
return pd.DataFrame(data)
|
|
694
|
-
|
|
792
|
+
|
|
695
793
|
|
|
696
794
|
@dataclass
|
|
697
795
|
class RedTeamRiskMitigationGuardrailsPolicyConfig(BaseDTO):
|
|
@@ -735,21 +833,21 @@ class RedTeamRiskMitigationGuardrailsPolicyResponse(BaseDTO):
|
|
|
735
833
|
|
|
736
834
|
def to_dict(self) -> dict:
|
|
737
835
|
policy_dict = self.guardrails_policy.to_dict()
|
|
738
|
-
|
|
836
|
+
|
|
739
837
|
# Remove detector entries that are disabled and have no other config
|
|
740
838
|
final_policy_dict = {}
|
|
741
839
|
for key, value in policy_dict.items():
|
|
742
840
|
if isinstance(value, dict):
|
|
743
841
|
# Check if 'enabled' is the only key and its value is False
|
|
744
|
-
if list(value.keys()) == [
|
|
842
|
+
if list(value.keys()) == ["enabled"] and not value["enabled"]:
|
|
745
843
|
continue
|
|
746
844
|
# Check for empty detectors that only have 'enabled': False
|
|
747
845
|
if not value.get("enabled", True) and len(value) == 1:
|
|
748
846
|
continue
|
|
749
847
|
# check for other empty values
|
|
750
|
-
if not any(v for k, v in value.items() if k !=
|
|
751
|
-
|
|
752
|
-
|
|
848
|
+
if not any(v for k, v in value.items() if k != "enabled"):
|
|
849
|
+
if not value.get("enabled"):
|
|
850
|
+
continue
|
|
753
851
|
final_policy_dict[key] = value
|
|
754
852
|
|
|
755
853
|
return {
|
|
@@ -804,21 +902,18 @@ class RedTeamRiskMitigationSystemPromptResponse(BaseDTO):
|
|
|
804
902
|
"message": self.message,
|
|
805
903
|
}
|
|
806
904
|
|
|
905
|
+
|
|
807
906
|
@dataclass
|
|
808
907
|
class RedTeamKeyFinding(BaseDTO):
|
|
809
908
|
text: str
|
|
810
909
|
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
811
|
-
|
|
910
|
+
|
|
812
911
|
@classmethod
|
|
813
912
|
def from_dict(cls, data: Dict[str, Any]) -> "RedTeamKeyFinding":
|
|
814
|
-
return cls(
|
|
815
|
-
|
|
816
|
-
)
|
|
817
|
-
|
|
913
|
+
return cls(text=data.get("text", ""))
|
|
914
|
+
|
|
818
915
|
def to_dict(self) -> Dict[str, Any]:
|
|
819
|
-
result = {
|
|
820
|
-
"text": self.text
|
|
821
|
-
}
|
|
916
|
+
result = {"text": self.text}
|
|
822
917
|
result.update(self._extra_fields)
|
|
823
918
|
return result
|
|
824
919
|
|
|
@@ -828,26 +923,240 @@ class RedTeamFindingsResponse(BaseDTO):
|
|
|
828
923
|
key_findings: List[RedTeamKeyFinding] = field(default_factory=list)
|
|
829
924
|
message: str = ""
|
|
830
925
|
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
831
|
-
|
|
926
|
+
|
|
832
927
|
@classmethod
|
|
833
928
|
def from_dict(cls, data: Dict[str, Any]) -> "RedTeamFindingsResponse":
|
|
834
929
|
key_findings_data = data.get("key_findings", [])
|
|
835
|
-
key_findings = [
|
|
836
|
-
|
|
930
|
+
key_findings = [
|
|
931
|
+
RedTeamKeyFinding.from_dict(finding) for finding in key_findings_data
|
|
932
|
+
]
|
|
933
|
+
|
|
934
|
+
return cls(key_findings=key_findings, message=data.get("message", ""))
|
|
935
|
+
|
|
936
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
937
|
+
result = {
|
|
938
|
+
"key_findings": [finding.to_dict() for finding in self.key_findings],
|
|
939
|
+
"message": self.message,
|
|
940
|
+
}
|
|
941
|
+
result.update(self._extra_fields)
|
|
942
|
+
return result
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
@dataclass
|
|
946
|
+
class RedTeamDownloadLinkResponse(BaseDTO):
|
|
947
|
+
link: str = ""
|
|
948
|
+
expiry: str = ""
|
|
949
|
+
expires_at: str = ""
|
|
950
|
+
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
951
|
+
|
|
952
|
+
@classmethod
|
|
953
|
+
def from_dict(cls, data: Dict[str, Any]) -> "RedTeamDownloadLinkResponse":
|
|
837
954
|
return cls(
|
|
838
|
-
|
|
839
|
-
|
|
955
|
+
link=data.get("link", ""),
|
|
956
|
+
expiry=data.get("expiry", ""),
|
|
957
|
+
expires_at=data.get("expires_at", ""),
|
|
840
958
|
)
|
|
841
|
-
|
|
959
|
+
|
|
842
960
|
def to_dict(self) -> Dict[str, Any]:
|
|
843
961
|
result = {
|
|
844
|
-
"
|
|
845
|
-
"
|
|
962
|
+
"link": self.link,
|
|
963
|
+
"expiry": self.expiry,
|
|
964
|
+
"expires_at": self.expires_at,
|
|
846
965
|
}
|
|
847
966
|
result.update(self._extra_fields)
|
|
848
967
|
return result
|
|
849
968
|
|
|
850
969
|
|
|
970
|
+
# V3 Attack Methods and Test Configurations
|
|
971
|
+
@dataclass
|
|
972
|
+
class AttackMethodsV3(BaseDTO):
|
|
973
|
+
"""
|
|
974
|
+
V3 format for attack methods with nested structure:
|
|
975
|
+
{
|
|
976
|
+
"method_category": {
|
|
977
|
+
"method_name": {
|
|
978
|
+
"params": {}
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
}
|
|
982
|
+
"""
|
|
983
|
+
|
|
984
|
+
_data: Dict[str, Dict[str, Dict[str, Any]]] = field(default_factory=dict)
|
|
985
|
+
|
|
986
|
+
def to_dict(self) -> dict:
|
|
987
|
+
return self._data
|
|
988
|
+
|
|
989
|
+
@classmethod
|
|
990
|
+
def from_dict(cls, data: dict):
|
|
991
|
+
return cls(_data=data)
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
@dataclass
|
|
995
|
+
class TestConfigV3(BaseDTO):
|
|
996
|
+
sample_percentage: int = 5
|
|
997
|
+
attack_methods: AttackMethodsV3 = field(default_factory=AttackMethodsV3)
|
|
998
|
+
|
|
999
|
+
def to_dict(self) -> dict:
|
|
1000
|
+
return {
|
|
1001
|
+
"sample_percentage": self.sample_percentage,
|
|
1002
|
+
"attack_methods": self.attack_methods.to_dict(),
|
|
1003
|
+
}
|
|
1004
|
+
|
|
1005
|
+
@classmethod
|
|
1006
|
+
def from_dict(cls, data: dict):
|
|
1007
|
+
attack_methods = AttackMethodsV3.from_dict(data.get("attack_methods", {}))
|
|
1008
|
+
return cls(
|
|
1009
|
+
sample_percentage=data.get("sample_percentage", 5),
|
|
1010
|
+
attack_methods=attack_methods,
|
|
1011
|
+
)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
@dataclass
|
|
1015
|
+
class RedTeamTestConfigurationsV3(BaseDTO):
|
|
1016
|
+
"""V3 format for red team test configurations with nested attack methods"""
|
|
1017
|
+
|
|
1018
|
+
version: str = "3.0"
|
|
1019
|
+
# Basic tests
|
|
1020
|
+
bias_test: TestConfigV3 = field(default=None)
|
|
1021
|
+
cbrn_test: TestConfigV3 = field(default=None)
|
|
1022
|
+
csem_test: TestConfigV3 = field(default=None)
|
|
1023
|
+
insecure_code_test: TestConfigV3 = field(default=None)
|
|
1024
|
+
toxicity_test: TestConfigV3 = field(default=None)
|
|
1025
|
+
harmful_test: TestConfigV3 = field(default=None)
|
|
1026
|
+
pii_test: TestConfigV3 = field(default=None)
|
|
1027
|
+
copyright_test: TestConfigV3 = field(default=None)
|
|
1028
|
+
misinformation_test: TestConfigV3 = field(default=None)
|
|
1029
|
+
system_prompt_extractions_test: TestConfigV3 = field(default=None)
|
|
1030
|
+
sponge_test: TestConfigV3 = field(default=None)
|
|
1031
|
+
# Advanced tests
|
|
1032
|
+
adv_info_test: TestConfigV3 = field(default=None)
|
|
1033
|
+
adv_bias_test: TestConfigV3 = field(default=None)
|
|
1034
|
+
adv_tool_test: TestConfigV3 = field(default=None)
|
|
1035
|
+
adv_command_test: TestConfigV3 = field(default=None)
|
|
1036
|
+
adv_pii_test: TestConfigV3 = field(default=None)
|
|
1037
|
+
adv_competitor_test: TestConfigV3 = field(default=None)
|
|
1038
|
+
# Custom tests
|
|
1039
|
+
custom_test: TestConfigV3 = field(default=None)
|
|
1040
|
+
# Agents tests
|
|
1041
|
+
alignment_and_governance_test: TestConfigV3 = field(default=None)
|
|
1042
|
+
input_and_content_integrity_test: TestConfigV3 = field(default=None)
|
|
1043
|
+
infrastructure_and_integration_test: TestConfigV3 = field(default=None)
|
|
1044
|
+
security_and_privacy_test: TestConfigV3 = field(default=None)
|
|
1045
|
+
human_factors_and_societal_impact_test: TestConfigV3 = field(default=None)
|
|
1046
|
+
access_control_test: TestConfigV3 = field(default=None)
|
|
1047
|
+
physical_and_actuation_safety_test: TestConfigV3 = field(default=None)
|
|
1048
|
+
reliability_and_monitoring_test: TestConfigV3 = field(default=None)
|
|
1049
|
+
governance_test: TestConfigV3 = field(default=None)
|
|
1050
|
+
agent_output_quality_test: TestConfigV3 = field(default=None)
|
|
1051
|
+
tool_misuse_test: TestConfigV3 = field(default=None)
|
|
1052
|
+
privacy_test: TestConfigV3 = field(default=None)
|
|
1053
|
+
reliability_and_observability_test: TestConfigV3 = field(default=None)
|
|
1054
|
+
agent_behaviour_test: TestConfigV3 = field(default=None)
|
|
1055
|
+
access_control_and_permissions_test: TestConfigV3 = field(default=None)
|
|
1056
|
+
tool_extraction_test: TestConfigV3 = field(default=None)
|
|
1057
|
+
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
1058
|
+
|
|
1059
|
+
@classmethod
|
|
1060
|
+
def from_dict(cls, data: dict):
|
|
1061
|
+
return cls(
|
|
1062
|
+
**{
|
|
1063
|
+
k: TestConfigV3.from_dict(v) if isinstance(v, dict) else v
|
|
1064
|
+
for k, v in data.items()
|
|
1065
|
+
}
|
|
1066
|
+
)
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
@dataclass
|
|
1070
|
+
class RedTeamCustomConfigV3(BaseDTO):
|
|
1071
|
+
test_name: str = "Test Name"
|
|
1072
|
+
frameworks: List[str] = field(default_factory=list)
|
|
1073
|
+
|
|
1074
|
+
redteam_test_configurations: RedTeamTestConfigurationsV3 = field(
|
|
1075
|
+
default_factory=RedTeamTestConfigurationsV3
|
|
1076
|
+
)
|
|
1077
|
+
dataset_configuration: Optional[DatasetConfig] = None
|
|
1078
|
+
endpoint_configuration: Optional[ModelConfig] = None
|
|
1079
|
+
|
|
1080
|
+
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
1081
|
+
|
|
1082
|
+
def to_dict(self) -> dict:
|
|
1083
|
+
d = asdict(self)
|
|
1084
|
+
d["redteam_test_configurations"] = self.redteam_test_configurations.to_dict()
|
|
1085
|
+
if self.dataset_configuration is not None:
|
|
1086
|
+
d["dataset_configuration"] = self.dataset_configuration.to_dict()
|
|
1087
|
+
else:
|
|
1088
|
+
d.pop("dataset_configuration", None)
|
|
1089
|
+
if self.endpoint_configuration is not None:
|
|
1090
|
+
d["endpoint_configuration"] = self.endpoint_configuration.to_dict()
|
|
1091
|
+
else:
|
|
1092
|
+
d.pop("endpoint_configuration", None)
|
|
1093
|
+
return d
|
|
1094
|
+
|
|
1095
|
+
@classmethod
|
|
1096
|
+
def from_dict(cls, data: dict):
|
|
1097
|
+
data = data.copy()
|
|
1098
|
+
test_configs = RedTeamTestConfigurationsV3.from_dict(
|
|
1099
|
+
data.pop("redteam_test_configurations", {})
|
|
1100
|
+
)
|
|
1101
|
+
dataset_config = None
|
|
1102
|
+
if "dataset_configuration" in data and data["dataset_configuration"]:
|
|
1103
|
+
dataset_config = DatasetConfig.from_dict(data.pop("dataset_configuration"))
|
|
1104
|
+
else:
|
|
1105
|
+
data.pop("dataset_configuration", None)
|
|
1106
|
+
|
|
1107
|
+
endpoint_config = None
|
|
1108
|
+
if "endpoint_configuration" in data and data["endpoint_configuration"]:
|
|
1109
|
+
endpoint_config = ModelConfig.from_dict(data.pop("endpoint_configuration"))
|
|
1110
|
+
else:
|
|
1111
|
+
data.pop("endpoint_configuration", None)
|
|
1112
|
+
|
|
1113
|
+
return cls(
|
|
1114
|
+
**data,
|
|
1115
|
+
redteam_test_configurations=test_configs,
|
|
1116
|
+
dataset_configuration=dataset_config,
|
|
1117
|
+
endpoint_configuration=endpoint_config,
|
|
1118
|
+
)
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
@dataclass
|
|
1122
|
+
class RedTeamCustomConfigWithSavedModelV3(BaseDTO):
|
|
1123
|
+
test_name: str = "Test Name"
|
|
1124
|
+
frameworks: List[str] = field(default_factory=list)
|
|
1125
|
+
|
|
1126
|
+
redteam_test_configurations: RedTeamTestConfigurationsV3 = field(
|
|
1127
|
+
default_factory=RedTeamTestConfigurationsV3
|
|
1128
|
+
)
|
|
1129
|
+
dataset_configuration: Optional[DatasetConfig] = None
|
|
1130
|
+
|
|
1131
|
+
_extra_fields: Dict[str, Any] = field(default_factory=dict)
|
|
1132
|
+
|
|
1133
|
+
def to_dict(self) -> dict:
|
|
1134
|
+
d = asdict(self)
|
|
1135
|
+
d["redteam_test_configurations"] = self.redteam_test_configurations.to_dict()
|
|
1136
|
+
if self.dataset_configuration is not None:
|
|
1137
|
+
d["dataset_configuration"] = self.dataset_configuration.to_dict()
|
|
1138
|
+
else:
|
|
1139
|
+
d.pop("dataset_configuration", None)
|
|
1140
|
+
return d
|
|
1141
|
+
|
|
1142
|
+
@classmethod
|
|
1143
|
+
def from_dict(cls, data: dict):
|
|
1144
|
+
data = data.copy()
|
|
1145
|
+
test_configs = RedTeamTestConfigurationsV3.from_dict(
|
|
1146
|
+
data.pop("redteam_test_configurations", {})
|
|
1147
|
+
)
|
|
1148
|
+
dataset_config = None
|
|
1149
|
+
if "dataset_configuration" in data and data["dataset_configuration"]:
|
|
1150
|
+
dataset_config = DatasetConfig.from_dict(data.pop("dataset_configuration"))
|
|
1151
|
+
else:
|
|
1152
|
+
data.pop("dataset_configuration", None)
|
|
1153
|
+
|
|
1154
|
+
return cls(
|
|
1155
|
+
**data,
|
|
1156
|
+
redteam_test_configurations=test_configs,
|
|
1157
|
+
dataset_configuration=dataset_config,
|
|
1158
|
+
)
|
|
1159
|
+
|
|
851
1160
|
|
|
852
1161
|
# Default configurations
|
|
853
1162
|
DEFAULT_REDTEAM_CONFIG = RedTeamConfig()
|
|
@@ -855,3 +1164,8 @@ DEFAULT_REDTEAM_CONFIG_WITH_SAVED_MODEL = RedTeamConfigWithSavedModel()
|
|
|
855
1164
|
|
|
856
1165
|
DEFAULT_CUSTOM_REDTEAM_CONFIG = RedTeamCustomConfig()
|
|
857
1166
|
DEFAULT_CUSTOM_REDTEAM_CONFIG_WITH_SAVED_MODEL = RedTeamCustomConfigWithSavedModel()
|
|
1167
|
+
|
|
1168
|
+
DEFAULT_CUSTOM_REDTEAM_CONFIG_V3 = RedTeamCustomConfigV3()
|
|
1169
|
+
DEFAULT_CUSTOM_REDTEAM_CONFIG_WITH_SAVED_MODEL_V3 = (
|
|
1170
|
+
RedTeamCustomConfigWithSavedModelV3()
|
|
1171
|
+
)
|