fable-engine 1.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- fable_compressor.py +356 -0
- fable_engine/__init__.py +1 -0
- fable_engine/actions/__init__.py +291 -0
- fable_engine/actions/cas.py +182 -0
- fable_engine/actions/deliberation.py +523 -0
- fable_engine/actions/fleet.py +807 -0
- fable_engine/actions/lifecycle.py +298 -0
- fable_engine/actions/scrapers.py +116 -0
- fable_engine/actions/system3.py +815 -0
- fable_engine/browser.py +824 -0
- fable_engine/cas.py +974 -0
- fable_engine/fable_session.json +510 -0
- fable_engine/guards.py +283 -0
- fable_engine/schema.py +714 -0
- fable_engine/scrapers/__init__.py +32 -0
- fable_engine/scrapers/arxiv.py +115 -0
- fable_engine/scrapers/base.py +386 -0
- fable_engine/scrapers/github.py +129 -0
- fable_engine/scrapers/reddit.py +154 -0
- fable_engine/scrapers/web.py +120 -0
- fable_engine/scrapers/x.py +125 -0
- fable_engine/scrapers/youtube.py +132 -0
- fable_engine/server.py +414 -0
- fable_engine/session.py +1819 -0
- fable_engine/test_server.py +1362 -0
- fable_engine/updater.py +541 -0
- fable_engine-1.3.1.dist-info/LICENSE +22 -0
- fable_engine-1.3.1.dist-info/METADATA +173 -0
- fable_engine-1.3.1.dist-info/RECORD +104 -0
- fable_engine-1.3.1.dist-info/WHEEL +5 -0
- fable_engine-1.3.1.dist-info/entry_points.txt +5 -0
- fable_engine-1.3.1.dist-info/top_level.txt +6 -0
- fable_mode/__init__.py +3 -0
- fable_mode/__main__.py +4 -0
- fable_mode/adapters.py +1014 -0
- fable_mode/installer.py +553 -0
- fable_mode/launcher.py +437 -0
- fable_mode/manifest.py +142 -0
- fable_mode/resources.json +114 -0
- fable_mode/safety.py +103 -0
- fable_mode_entry.py +10 -0
- fable_v2/__init__.py +146 -0
- fable_v2/adapters.py +151 -0
- fable_v2/coder_fleet/__init__.py +100 -0
- fable_v2/coder_fleet/ast_tools.py +158 -0
- fable_v2/coder_fleet/compute.py +199 -0
- fable_v2/coder_fleet/design_engine.py +1316 -0
- fable_v2/coder_fleet/diagnostics.py +293 -0
- fable_v2/coder_fleet/fleet_dispatcher.py +214 -0
- fable_v2/coder_fleet/mock_auditor.py +306 -0
- fable_v2/coder_fleet/mutation.py +216 -0
- fable_v2/coder_fleet/property_oracle.py +260 -0
- fable_v2/coder_fleet/receipt_attestor.py +122 -0
- fable_v2/coder_fleet/red_team_swarm.py +908 -0
- fable_v2/coder_fleet/test_harness.py +198 -0
- fable_v2/coder_fleet/vector_engine.py +1287 -0
- fable_v2/coder_fleet/visual.py +357 -0
- fable_v2/coder_fleet/workspace.py +153 -0
- fable_v2/cortical/__init__.py +20 -0
- fable_v2/cortical/plasticity_engine.py +992 -0
- fable_v2/execution_broker.py +811 -0
- fable_v2/proof_engine.py +1141 -0
- fable_v2/protocol.py +485 -0
- fable_v2/runtime.py +1010 -0
- fable_v2/system3/__init__.py +204 -0
- fable_v2/system3/causal.py +558 -0
- fable_v2/system3/dialectical.py +577 -0
- fable_v2/system3/evolution.py +503 -0
- fable_v2/system3/executive.py +338 -0
- fable_v2/system3/free_energy.py +479 -0
- fable_v2/system3/hyperbolic.py +555 -0
- fable_v2/system3/induction.py +336 -0
- fable_v2/system3/kripke.py +548 -0
- fable_v2/system3/oracle.py +745 -0
- fable_v2/verifiers.py +72 -0
- tests/__init__.py +1 -0
- tests/test_anti_loop_circuit_breaker.py +64 -0
- tests/test_auto_updater.py +407 -0
- tests/test_coder_fleet.py +535 -0
- tests/test_delegation_compiler.py +54 -0
- tests/test_descriptor_boundaries.py +126 -0
- tests/test_design_engine.py +603 -0
- tests/test_epistemic_evidence_validator.py +66 -0
- tests/test_execution_broker.py +233 -0
- tests/test_fable_v2.py +406 -0
- tests/test_fleet_transitions.py +116 -0
- tests/test_fsm_redteam_evolution.py +406 -0
- tests/test_goal_rubric_and_pipeline.py +367 -0
- tests/test_hebbian_plasticity.py +585 -0
- tests/test_packaging_runtime.py +194 -0
- tests/test_proof_engine.py +259 -0
- tests/test_red_team_swarm.py +645 -0
- tests/test_redteam_remediation.py +169 -0
- tests/test_registration_transaction.py +375 -0
- tests/test_requested_regressions.py +467 -0
- tests/test_scrapers.py +370 -0
- tests/test_server_actions.py +93 -0
- tests/test_server_frontier_actions.py +269 -0
- tests/test_server_protocol.py +88 -0
- tests/test_stealth_browser.py +970 -0
- tests/test_system3.py +381 -0
- tests/test_system3_deep_integration.py +385 -0
- tests/test_system3_frontier.py +436 -0
- tests/test_vector_engine.py +608 -0
|
@@ -0,0 +1,908 @@
|
|
|
1
|
+
"""Modular Fable Part 1: Adversarial Code Review Swarm (Project Glasswing Red Team Loop).
|
|
2
|
+
|
|
3
|
+
Production-grade Red Team Swarm engine providing:
|
|
4
|
+
- 5 Attack Vectors: Chaos Environment, Byzantine Payload, Concurrency Race, Resource Exhaustion, State Invariant
|
|
5
|
+
- BreakScenario, BreakFinding, and RedTeamBreakageReport dataclasses
|
|
6
|
+
- RedTeamSwarm engine executing counterfactual "What if?" attack probes
|
|
7
|
+
- Automated Ping-Pong Hardening & Remediation Verification protocol
|
|
8
|
+
"""
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import datetime
|
|
12
|
+
import inspect
|
|
13
|
+
import json
|
|
14
|
+
import os
|
|
15
|
+
import threading
|
|
16
|
+
import time
|
|
17
|
+
import traceback
|
|
18
|
+
import uuid
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from enum import Enum
|
|
21
|
+
from typing import Any, Callable, Optional, Union
|
|
22
|
+
|
|
23
|
+
from .mock_auditor import MockAuditorEngine
|
|
24
|
+
from .property_oracle import PropertyOracleEngine
|
|
25
|
+
from .test_harness import TestHarnessEngine
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class AttackVector(str, Enum):
|
|
29
|
+
"""The 5 Core Attack Vectors for Adversarial Code Review."""
|
|
30
|
+
|
|
31
|
+
CHAOS_ENVIRONMENT = "chaos_environment"
|
|
32
|
+
BYZANTINE_PAYLOAD = "byzantine_payload"
|
|
33
|
+
CONCURRENCY_RACE = "concurrency_race"
|
|
34
|
+
RESOURCE_EXHAUSTION = "resource_exhaustion"
|
|
35
|
+
STATE_INVARIANT = "state_invariant"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class BreakScenario:
|
|
40
|
+
"""An adversarial scenario probing what will happen under extreme conditions."""
|
|
41
|
+
|
|
42
|
+
scenario_id: str
|
|
43
|
+
vector: AttackVector
|
|
44
|
+
hypothesis: str # "What will happen if X happened?"
|
|
45
|
+
expected_resilience: str = ""
|
|
46
|
+
attack_fn: Optional[Callable[..., Any]] = None
|
|
47
|
+
metadata: dict[str, Any] = field(default_factory=dict)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass
|
|
51
|
+
class BreakFinding:
|
|
52
|
+
"""The finding result from executing an adversarial probe against a target."""
|
|
53
|
+
|
|
54
|
+
scenario_id: str
|
|
55
|
+
vector: str
|
|
56
|
+
hypothesis: str
|
|
57
|
+
broken: bool
|
|
58
|
+
error_message: Optional[str] = None
|
|
59
|
+
traceback_snippet: Optional[str] = None
|
|
60
|
+
reproduction_code: Optional[str] = None
|
|
61
|
+
severity: str = "MEDIUM" # "CRITICAL", "HIGH", "MEDIUM", "LOW"
|
|
62
|
+
details: dict[str, Any] = field(default_factory=dict)
|
|
63
|
+
|
|
64
|
+
def to_dict(self) -> dict[str, Any]:
|
|
65
|
+
"""Serialize finding to dictionary."""
|
|
66
|
+
return {
|
|
67
|
+
"scenario_id": self.scenario_id,
|
|
68
|
+
"vector": self.vector,
|
|
69
|
+
"hypothesis": self.hypothesis,
|
|
70
|
+
"broken": self.broken,
|
|
71
|
+
"error_message": self.error_message,
|
|
72
|
+
"traceback_snippet": self.traceback_snippet,
|
|
73
|
+
"reproduction_code": self.reproduction_code,
|
|
74
|
+
"severity": self.severity,
|
|
75
|
+
"details": self.details,
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@dataclass
|
|
80
|
+
class RedTeamBreakageReport:
|
|
81
|
+
"""Comprehensive breakage and resilience report compiled by RedTeamSwarm."""
|
|
82
|
+
|
|
83
|
+
report_id: str
|
|
84
|
+
target_name: str
|
|
85
|
+
total_probes: int
|
|
86
|
+
broken_count: int
|
|
87
|
+
passed: bool
|
|
88
|
+
findings: list[BreakFinding]
|
|
89
|
+
created_at: str
|
|
90
|
+
remediation_directives: list[str] = field(default_factory=list)
|
|
91
|
+
|
|
92
|
+
def to_dict(self) -> dict[str, Any]:
|
|
93
|
+
"""Serialize report to dictionary."""
|
|
94
|
+
return {
|
|
95
|
+
"report_id": self.report_id,
|
|
96
|
+
"target_name": self.target_name,
|
|
97
|
+
"total_probes": self.total_probes,
|
|
98
|
+
"broken_count": self.broken_count,
|
|
99
|
+
"passed": self.passed,
|
|
100
|
+
"created_at": self.created_at,
|
|
101
|
+
"remediation_directives": list(self.remediation_directives),
|
|
102
|
+
"findings": [
|
|
103
|
+
{
|
|
104
|
+
"scenario_id": f.scenario_id,
|
|
105
|
+
"vector": f.vector,
|
|
106
|
+
"hypothesis": f.hypothesis,
|
|
107
|
+
"broken": f.broken,
|
|
108
|
+
"error_message": f.error_message,
|
|
109
|
+
"traceback_snippet": f.traceback_snippet,
|
|
110
|
+
"reproduction_code": f.reproduction_code,
|
|
111
|
+
"severity": f.severity,
|
|
112
|
+
"details": f.details,
|
|
113
|
+
}
|
|
114
|
+
for f in self.findings
|
|
115
|
+
],
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
@classmethod
|
|
119
|
+
def from_dict(cls, data: dict[str, Any]) -> RedTeamBreakageReport:
|
|
120
|
+
"""Construct RedTeamBreakageReport from dictionary."""
|
|
121
|
+
findings = [
|
|
122
|
+
BreakFinding(
|
|
123
|
+
scenario_id=item.get("scenario_id", ""),
|
|
124
|
+
vector=item.get("vector", AttackVector.CHAOS_ENVIRONMENT.value),
|
|
125
|
+
hypothesis=item.get("hypothesis", ""),
|
|
126
|
+
broken=item.get("broken", False),
|
|
127
|
+
error_message=item.get("error_message"),
|
|
128
|
+
traceback_snippet=item.get("traceback_snippet"),
|
|
129
|
+
reproduction_code=item.get("reproduction_code"),
|
|
130
|
+
severity=item.get("severity", "MEDIUM"),
|
|
131
|
+
details=item.get("details", {}),
|
|
132
|
+
)
|
|
133
|
+
for item in data.get("findings", [])
|
|
134
|
+
]
|
|
135
|
+
return cls(
|
|
136
|
+
report_id=data.get("report_id", f"report_{uuid.uuid4().hex[:8]}"),
|
|
137
|
+
target_name=data.get("target_name", "target"),
|
|
138
|
+
total_probes=data.get("total_probes", len(findings)),
|
|
139
|
+
broken_count=data.get("broken_count", sum(1 for f in findings if f.broken)),
|
|
140
|
+
passed=data.get("passed", sum(1 for f in findings if f.broken) == 0),
|
|
141
|
+
findings=findings,
|
|
142
|
+
created_at=data.get("created_at", datetime.datetime.now(datetime.timezone.utc).isoformat()),
|
|
143
|
+
remediation_directives=data.get("remediation_directives", []),
|
|
144
|
+
)
|
|
145
|
+
|
|
146
|
+
def to_markdown(self) -> str:
|
|
147
|
+
"""Render report as clean GitHub-flavored markdown with alerts and tables."""
|
|
148
|
+
lines: list[str] = []
|
|
149
|
+
|
|
150
|
+
if self.passed:
|
|
151
|
+
lines.append(f"# 🛡️ Adversarial Red Team Resilient Attestation: `{self.target_name}`")
|
|
152
|
+
else:
|
|
153
|
+
lines.append(f"# 🚨 Adversarial Red Team Breakage Report: `{self.target_name}`")
|
|
154
|
+
|
|
155
|
+
lines.append("")
|
|
156
|
+
lines.append(f"- **Report ID**: `{self.report_id}`")
|
|
157
|
+
lines.append(f"- **Target Name**: `{self.target_name}`")
|
|
158
|
+
lines.append(f"- **Total Probes**: `{self.total_probes}`")
|
|
159
|
+
lines.append(f"- **Broken Scenarios**: `{self.broken_count}`")
|
|
160
|
+
lines.append(f"- **Status**: {'🟢 **RESILIENT (PASSED)**' if self.passed else '🔴 **BREAKAGES DETECTED (FAILED)**'}")
|
|
161
|
+
lines.append(f"- **Created At**: `{self.created_at}`")
|
|
162
|
+
lines.append("")
|
|
163
|
+
|
|
164
|
+
if not self.passed:
|
|
165
|
+
lines.append("> [!CAUTION]")
|
|
166
|
+
lines.append(f"> Swarm attacks detected **{self.broken_count} breakages** in `{self.target_name}`!")
|
|
167
|
+
lines.append("> Subagent code changes are **REJECTED** until all remediation directives are implemented and verified.")
|
|
168
|
+
else:
|
|
169
|
+
lines.append("> [!NOTE]")
|
|
170
|
+
lines.append(f"> Target `{self.target_name}` survived all **{self.total_probes} adversarial probes** with zero unhandled breakages.")
|
|
171
|
+
lines.append("> Verified resilient against chaos, Byzantine inputs, concurrency races, resource limits, and state invariant attacks.")
|
|
172
|
+
|
|
173
|
+
lines.append("")
|
|
174
|
+
lines.append("### 📋 Probes & Findings Overview")
|
|
175
|
+
lines.append("")
|
|
176
|
+
lines.append("| Scenario ID | Vector | Hypothesis | Broken | Severity |")
|
|
177
|
+
lines.append("|---|---|---|:---:|:---:|")
|
|
178
|
+
|
|
179
|
+
for f in self.findings:
|
|
180
|
+
status_icon = "💥 **YES**" if f.broken else "✅ NO"
|
|
181
|
+
safe_hyp = f.hypothesis.replace("|", "\\|")
|
|
182
|
+
lines.append(f"| `{f.scenario_id}` | `{f.vector}` | {safe_hyp} | {status_icon} | `{f.severity}` |")
|
|
183
|
+
|
|
184
|
+
# Detail any breakages
|
|
185
|
+
broken_findings = [f for f in self.findings if f.broken]
|
|
186
|
+
if broken_findings:
|
|
187
|
+
lines.append("")
|
|
188
|
+
lines.append("### 💥 Breakage Diagnostic Details")
|
|
189
|
+
for bf in broken_findings:
|
|
190
|
+
lines.append("")
|
|
191
|
+
lines.append(f"#### 🔴 `{bf.scenario_id}` (`{bf.vector}`) - Severity: `{bf.severity}`")
|
|
192
|
+
lines.append(f"**Hypothesis**: {bf.hypothesis}")
|
|
193
|
+
if bf.error_message:
|
|
194
|
+
lines.append(f"**Observed Failure**: `{bf.error_message}`")
|
|
195
|
+
if bf.traceback_snippet:
|
|
196
|
+
lines.append("")
|
|
197
|
+
lines.append("```text")
|
|
198
|
+
lines.append(bf.traceback_snippet.strip())
|
|
199
|
+
lines.append("```")
|
|
200
|
+
if bf.reproduction_code:
|
|
201
|
+
lines.append("")
|
|
202
|
+
lines.append("**Reproduction Snippet**:")
|
|
203
|
+
lines.append("```python")
|
|
204
|
+
lines.append(bf.reproduction_code.strip())
|
|
205
|
+
lines.append("```")
|
|
206
|
+
|
|
207
|
+
# Remediation directives
|
|
208
|
+
lines.append("")
|
|
209
|
+
lines.append("### 🛠️ Remediation Directives")
|
|
210
|
+
if self.remediation_directives:
|
|
211
|
+
for idx, directive in enumerate(self.remediation_directives, 1):
|
|
212
|
+
lines.append(f"{idx}. {directive}")
|
|
213
|
+
else:
|
|
214
|
+
lines.append("- No remediation required. All adversarial probes passed.")
|
|
215
|
+
|
|
216
|
+
lines.append("")
|
|
217
|
+
return "\n".join(lines)
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
class RedTeamSwarm:
|
|
221
|
+
"""Adversarial Red Team Swarm executing counterfactual stress probes across 5 vectors.
|
|
222
|
+
|
|
223
|
+
RedTeamSwarm never executes source-code strings in-process. Source-code strings and
|
|
224
|
+
unexecutable targets are rejected as non-executable targets (passed=False). Dynamic source
|
|
225
|
+
execution, if required, must be provided through a separately sandboxed execution layer.
|
|
226
|
+
"""
|
|
227
|
+
|
|
228
|
+
def __init__(
|
|
229
|
+
self,
|
|
230
|
+
test_harness: Optional[TestHarnessEngine] = None,
|
|
231
|
+
mock_auditor: Optional[MockAuditorEngine] = None,
|
|
232
|
+
property_oracle: Optional[PropertyOracleEngine] = None,
|
|
233
|
+
plasticity_engine: Optional[Any] = None,
|
|
234
|
+
) -> None:
|
|
235
|
+
self.test_harness = test_harness or TestHarnessEngine()
|
|
236
|
+
self.mock_auditor = mock_auditor or MockAuditorEngine()
|
|
237
|
+
self.property_oracle = property_oracle or PropertyOracleEngine(test_harness=self.test_harness)
|
|
238
|
+
if plasticity_engine is not None:
|
|
239
|
+
self.plasticity_engine = plasticity_engine
|
|
240
|
+
else:
|
|
241
|
+
try:
|
|
242
|
+
from ..cortical.plasticity_engine import HebbianPlasticityEngine
|
|
243
|
+
self.plasticity_engine = HebbianPlasticityEngine()
|
|
244
|
+
except Exception:
|
|
245
|
+
try:
|
|
246
|
+
from fable_v2.cortical.plasticity_engine import HebbianPlasticityEngine
|
|
247
|
+
self.plasticity_engine = HebbianPlasticityEngine()
|
|
248
|
+
except Exception:
|
|
249
|
+
self.plasticity_engine = None
|
|
250
|
+
|
|
251
|
+
def _resolve_callable(self, target: Any) -> Optional[Callable[..., Any]]:
|
|
252
|
+
"""Resolves target callable from function or class object.
|
|
253
|
+
|
|
254
|
+
RedTeamSwarm never executes source-code strings in-process. Source-code strings
|
|
255
|
+
are rejected as non-executable targets. Dynamic source execution, if required,
|
|
256
|
+
must be provided through a separately sandboxed execution layer.
|
|
257
|
+
"""
|
|
258
|
+
if callable(target):
|
|
259
|
+
return target
|
|
260
|
+
return None
|
|
261
|
+
|
|
262
|
+
def generate_break_scenarios(
|
|
263
|
+
self,
|
|
264
|
+
target_name: str = "target",
|
|
265
|
+
code_snippet: Optional[str] = None,
|
|
266
|
+
custom_hypotheses: Optional[list[str]] = None,
|
|
267
|
+
) -> list[BreakScenario]:
|
|
268
|
+
"""Generates comprehensive 'what if' break scenarios across all 5 attack vectors.
|
|
269
|
+
|
|
270
|
+
Includes standard stress vectors and any user-specified custom hypotheses.
|
|
271
|
+
"""
|
|
272
|
+
scenarios: list[BreakScenario] = []
|
|
273
|
+
|
|
274
|
+
# Vector 1: CHAOS_ENVIRONMENT
|
|
275
|
+
def _chaos_missing_path(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
276
|
+
if fn is None:
|
|
277
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
278
|
+
# Probe with non-existent / unlinked file path
|
|
279
|
+
return fn("/nonexistent/fable_chaos_probe_file.tmp")
|
|
280
|
+
|
|
281
|
+
scenarios.append(
|
|
282
|
+
BreakScenario(
|
|
283
|
+
scenario_id=f"{target_name}_chaos_01_missing_path",
|
|
284
|
+
vector=AttackVector.CHAOS_ENVIRONMENT,
|
|
285
|
+
hypothesis="What will happen if targeted files or temporary paths are missing, unlinked, or raise FileNotFoundError?",
|
|
286
|
+
expected_resilience="Gracefully catches missing path or raises structured domain error without unhandled crash",
|
|
287
|
+
attack_fn=_chaos_missing_path,
|
|
288
|
+
metadata={"probe_type": "missing_path"},
|
|
289
|
+
)
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
def _chaos_corrupt_env(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
293
|
+
if fn is None:
|
|
294
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
295
|
+
return fn("")
|
|
296
|
+
|
|
297
|
+
scenarios.append(
|
|
298
|
+
BreakScenario(
|
|
299
|
+
scenario_id=f"{target_name}_chaos_02_empty_or_corrupt_env",
|
|
300
|
+
vector=AttackVector.CHAOS_ENVIRONMENT,
|
|
301
|
+
hypothesis="What will happen if configuration, environment strings, or descriptors are empty, corrupt, or None?",
|
|
302
|
+
expected_resilience="Handles empty or missing environment gracefully",
|
|
303
|
+
attack_fn=_chaos_corrupt_env,
|
|
304
|
+
metadata={"probe_type": "empty_input"},
|
|
305
|
+
)
|
|
306
|
+
)
|
|
307
|
+
|
|
308
|
+
# Vector 2: BYZANTINE_PAYLOAD
|
|
309
|
+
def _byzantine_null_bytes(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
310
|
+
if fn is None:
|
|
311
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
312
|
+
payload = "probe\x00hostile\x00injection\r\n\t"
|
|
313
|
+
return fn(payload)
|
|
314
|
+
|
|
315
|
+
scenarios.append(
|
|
316
|
+
BreakScenario(
|
|
317
|
+
scenario_id=f"{target_name}_byzantine_01_null_bytes",
|
|
318
|
+
vector=AttackVector.BYZANTINE_PAYLOAD,
|
|
319
|
+
hypothesis="What will happen if input strings contain embedded null bytes (\\x00), escape controls, or surrogates?",
|
|
320
|
+
expected_resilience="Properly sanitizes or rejects null-byte payloads without unhandled exception",
|
|
321
|
+
attack_fn=_byzantine_null_bytes,
|
|
322
|
+
metadata={"reproduction": "target_fn('probe\\x00hostile\\x00injection\\r\\n\\t')"},
|
|
323
|
+
)
|
|
324
|
+
)
|
|
325
|
+
|
|
326
|
+
def _byzantine_deep_nesting(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
327
|
+
if fn is None:
|
|
328
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
329
|
+
# 60 levels of nested dictionaries
|
|
330
|
+
nested: dict[str, Any] = {"leaf": 42}
|
|
331
|
+
for _ in range(60):
|
|
332
|
+
nested = {"nest": nested}
|
|
333
|
+
return fn(nested)
|
|
334
|
+
|
|
335
|
+
scenarios.append(
|
|
336
|
+
BreakScenario(
|
|
337
|
+
scenario_id=f"{target_name}_byzantine_02_deep_nesting",
|
|
338
|
+
vector=AttackVector.BYZANTINE_PAYLOAD,
|
|
339
|
+
hypothesis="What will happen if input payload contains deeply recursive nested dictionaries (60+ levels)?",
|
|
340
|
+
expected_resilience="Traverses safely without triggering RecursionError or stack overflow",
|
|
341
|
+
attack_fn=_byzantine_deep_nesting,
|
|
342
|
+
metadata={"reproduction": "nested = {'leaf': 42}; [nested := {'nest': nested} for _ in range(60)]; target_fn(nested)"},
|
|
343
|
+
)
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
def _byzantine_type_confusion(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
347
|
+
if fn is None:
|
|
348
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
349
|
+
return fn(None)
|
|
350
|
+
|
|
351
|
+
scenarios.append(
|
|
352
|
+
BreakScenario(
|
|
353
|
+
scenario_id=f"{target_name}_byzantine_03_type_confusion",
|
|
354
|
+
vector=AttackVector.BYZANTINE_PAYLOAD,
|
|
355
|
+
hypothesis="What will happen if None or an inverted data type is passed into the target callable?",
|
|
356
|
+
expected_resilience="Validates arguments and rejects None/incompatible types without raw AttributeError",
|
|
357
|
+
attack_fn=_byzantine_type_confusion,
|
|
358
|
+
metadata={"reproduction": "target_fn(None)"},
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
def _byzantine_extreme_numbers(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
363
|
+
if fn is None:
|
|
364
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
365
|
+
# Nan, Inf, negative zero, huge int
|
|
366
|
+
return fn(float("nan"))
|
|
367
|
+
|
|
368
|
+
scenarios.append(
|
|
369
|
+
BreakScenario(
|
|
370
|
+
scenario_id=f"{target_name}_byzantine_04_extreme_numbers",
|
|
371
|
+
vector=AttackVector.BYZANTINE_PAYLOAD,
|
|
372
|
+
hypothesis="What will happen if numeric inputs are NaN, Infinity, -0.0, or exceed 2**64?",
|
|
373
|
+
expected_resilience="Handles IEEE-754 special values and huge numbers safely",
|
|
374
|
+
attack_fn=_byzantine_extreme_numbers,
|
|
375
|
+
metadata={"reproduction": "target_fn(float('nan'))"},
|
|
376
|
+
)
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
# Vector 3: CONCURRENCY_RACE
|
|
380
|
+
def _concurrency_multithreaded_burst(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
381
|
+
if fn is None:
|
|
382
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
383
|
+
errors: list[str] = []
|
|
384
|
+
threads: list[threading.Thread] = []
|
|
385
|
+
try:
|
|
386
|
+
sig = inspect.signature(fn)
|
|
387
|
+
has_params = len(sig.parameters) > 0
|
|
388
|
+
except Exception:
|
|
389
|
+
has_params = True
|
|
390
|
+
|
|
391
|
+
def _worker() -> None:
|
|
392
|
+
for _ in range(25):
|
|
393
|
+
try:
|
|
394
|
+
if has_params:
|
|
395
|
+
fn("concurrency_probe_payload")
|
|
396
|
+
else:
|
|
397
|
+
fn()
|
|
398
|
+
except Exception as e:
|
|
399
|
+
errors.append(f"{type(e).__name__}: {e}")
|
|
400
|
+
break
|
|
401
|
+
|
|
402
|
+
for _ in range(6):
|
|
403
|
+
t = threading.Thread(target=_worker, daemon=True)
|
|
404
|
+
threads.append(t)
|
|
405
|
+
t.start()
|
|
406
|
+
|
|
407
|
+
for t in threads:
|
|
408
|
+
t.join(timeout=1.0)
|
|
409
|
+
|
|
410
|
+
if errors:
|
|
411
|
+
return {"broken": True, "error": f"Concurrency collision detected in thread burst: {errors[0]}"}
|
|
412
|
+
return {"broken": False}
|
|
413
|
+
|
|
414
|
+
scenarios.append(
|
|
415
|
+
BreakScenario(
|
|
416
|
+
scenario_id=f"{target_name}_concurrency_01_race_burst",
|
|
417
|
+
vector=AttackVector.CONCURRENCY_RACE,
|
|
418
|
+
hypothesis="What will happen if 6 concurrent threads invoke the target simultaneously across 25 iterations?",
|
|
419
|
+
expected_resilience="Thread-safe execution with zero race collisions or lost updates",
|
|
420
|
+
attack_fn=_concurrency_multithreaded_burst,
|
|
421
|
+
metadata={"reproduction": "Deploy 6 threads executing target_fn concurrently"},
|
|
422
|
+
)
|
|
423
|
+
)
|
|
424
|
+
|
|
425
|
+
# Vector 4: RESOURCE_EXHAUSTION
|
|
426
|
+
def _resource_massive_payload(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
427
|
+
if fn is None:
|
|
428
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
429
|
+
massive_str = "A" * 150_000
|
|
430
|
+
return fn(massive_str)
|
|
431
|
+
|
|
432
|
+
scenarios.append(
|
|
433
|
+
BreakScenario(
|
|
434
|
+
scenario_id=f"{target_name}_resource_01_massive_payload",
|
|
435
|
+
vector=AttackVector.RESOURCE_EXHAUSTION,
|
|
436
|
+
hypothesis="What will happen if the input payload is a massive 150,000-character payload?",
|
|
437
|
+
expected_resilience="Handles or bounds large payload without OOM or memory exhaustion",
|
|
438
|
+
attack_fn=_resource_massive_payload,
|
|
439
|
+
metadata={"reproduction": "target_fn('A' * 150_000)"},
|
|
440
|
+
)
|
|
441
|
+
)
|
|
442
|
+
|
|
443
|
+
def _resource_rapid_churn(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
444
|
+
if fn is None:
|
|
445
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
446
|
+
sig = inspect.signature(fn)
|
|
447
|
+
for _ in range(100):
|
|
448
|
+
if len(sig.parameters) == 0:
|
|
449
|
+
fn()
|
|
450
|
+
else:
|
|
451
|
+
fn("churn_sample")
|
|
452
|
+
return {"broken": False}
|
|
453
|
+
|
|
454
|
+
scenarios.append(
|
|
455
|
+
BreakScenario(
|
|
456
|
+
scenario_id=f"{target_name}_resource_02_rapid_churn",
|
|
457
|
+
vector=AttackVector.RESOURCE_EXHAUSTION,
|
|
458
|
+
hypothesis="What will happen if invoked in rapid back-to-back sequence 100 times?",
|
|
459
|
+
expected_resilience="No descriptor leakage or memory accumulation under rapid churn",
|
|
460
|
+
attack_fn=_resource_rapid_churn,
|
|
461
|
+
metadata={"reproduction": "[target_fn('churn_sample') for _ in range(100)]"},
|
|
462
|
+
)
|
|
463
|
+
)
|
|
464
|
+
|
|
465
|
+
# Vector 5: STATE_INVARIANT
|
|
466
|
+
def _state_invariant_idempotency(fn: Optional[Callable[..., Any]] = None) -> Any:
|
|
467
|
+
if fn is None:
|
|
468
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
469
|
+
sig = inspect.signature(fn)
|
|
470
|
+
if len(sig.parameters) == 0:
|
|
471
|
+
res1 = fn()
|
|
472
|
+
res2 = fn()
|
|
473
|
+
else:
|
|
474
|
+
res1 = fn("invariant_sample")
|
|
475
|
+
res2 = fn("invariant_sample")
|
|
476
|
+
return {"broken": False, "res1": str(res1)[:50], "res2": str(res2)[:50]}
|
|
477
|
+
|
|
478
|
+
scenarios.append(
|
|
479
|
+
BreakScenario(
|
|
480
|
+
scenario_id=f"{target_name}_state_01_idempotency",
|
|
481
|
+
vector=AttackVector.STATE_INVARIANT,
|
|
482
|
+
hypothesis="What will happen if identical operations are performed consecutively? Does target violate state consistency?",
|
|
483
|
+
expected_resilience="Idempotent or deterministic state transition without corruption",
|
|
484
|
+
attack_fn=_state_invariant_idempotency,
|
|
485
|
+
metadata={"reproduction": "r1 = target_fn('sample'); r2 = target_fn('sample')"},
|
|
486
|
+
)
|
|
487
|
+
)
|
|
488
|
+
|
|
489
|
+
# Process custom hypotheses
|
|
490
|
+
if custom_hypotheses:
|
|
491
|
+
for idx, hyp in enumerate(custom_hypotheses, 1):
|
|
492
|
+
hyp_clean = hyp.strip()
|
|
493
|
+
if not hyp_clean:
|
|
494
|
+
continue
|
|
495
|
+
|
|
496
|
+
h_lower = hyp_clean.lower()
|
|
497
|
+
if any(w in h_lower for w in ("race", "thread", "concurr", "lock", "toctou")):
|
|
498
|
+
v = AttackVector.CONCURRENCY_RACE
|
|
499
|
+
elif any(w in h_lower for w in ("byzantine", "payload", "null", "inject", "json", "nest")):
|
|
500
|
+
v = AttackVector.BYZANTINE_PAYLOAD
|
|
501
|
+
elif any(w in h_lower for w in ("memory", "cpu", "exhaust", "leak", "timeout", "resource", "massive")):
|
|
502
|
+
v = AttackVector.RESOURCE_EXHAUSTION
|
|
503
|
+
elif any(w in h_lower for w in ("invariant", "state", "idempotent", "order", "sequence", "balance")):
|
|
504
|
+
v = AttackVector.STATE_INVARIANT
|
|
505
|
+
else:
|
|
506
|
+
v = AttackVector.CHAOS_ENVIRONMENT
|
|
507
|
+
|
|
508
|
+
def _custom_attack_fn(fn: Optional[Callable[..., Any]] = None, h_text: str = hyp_clean) -> Any:
|
|
509
|
+
if fn is None:
|
|
510
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
511
|
+
sig = inspect.signature(fn)
|
|
512
|
+
if len(sig.parameters) == 0:
|
|
513
|
+
return fn()
|
|
514
|
+
return fn(f"probe:{h_text}")
|
|
515
|
+
|
|
516
|
+
scenarios.append(
|
|
517
|
+
BreakScenario(
|
|
518
|
+
scenario_id=f"{target_name}_custom_{idx:02d}",
|
|
519
|
+
vector=v,
|
|
520
|
+
hypothesis=hyp_clean,
|
|
521
|
+
expected_resilience=f"Resilient against hypothesis: {hyp_clean}",
|
|
522
|
+
attack_fn=_custom_attack_fn,
|
|
523
|
+
metadata={"custom": True},
|
|
524
|
+
)
|
|
525
|
+
)
|
|
526
|
+
|
|
527
|
+
return scenarios
|
|
528
|
+
|
|
529
|
+
def _derive_remediation_directives(self, findings: list[BreakFinding]) -> list[str]:
|
|
530
|
+
"""Derives actionable remediation directives based on broken findings."""
|
|
531
|
+
directives: set[str] = set()
|
|
532
|
+
|
|
533
|
+
for f in findings:
|
|
534
|
+
if not f.broken:
|
|
535
|
+
continue
|
|
536
|
+
|
|
537
|
+
v = f.vector
|
|
538
|
+
if v == AttackVector.BYZANTINE_PAYLOAD.value:
|
|
539
|
+
directives.add(
|
|
540
|
+
"Implement defensive input validation and sanitization: check types, reject or strip null bytes (\\x00), "
|
|
541
|
+
"enforce maximum recursion depth limits, and guard against special IEEE-754 numbers (NaN/Inf)."
|
|
542
|
+
)
|
|
543
|
+
elif v == AttackVector.CONCURRENCY_RACE.value:
|
|
544
|
+
directives.add(
|
|
545
|
+
"Enforce concurrency synchronization: wrap shared state updates with threading.Lock, "
|
|
546
|
+
"use atomic primitives, or make shared data structures read-only/immutable."
|
|
547
|
+
)
|
|
548
|
+
elif v == AttackVector.CHAOS_ENVIRONMENT.value:
|
|
549
|
+
directives.add(
|
|
550
|
+
"Implement fault-tolerant environment handling: wrap I/O and path operations in try/except "
|
|
551
|
+
"(FileNotFoundError, PermissionError, OSError) with graceful fallback defaults."
|
|
552
|
+
)
|
|
553
|
+
elif v == AttackVector.RESOURCE_EXHAUSTION.value:
|
|
554
|
+
directives.add(
|
|
555
|
+
"Apply resource bounding: enforce payload size limits (e.g., max 64KB), bounded loop counts, "
|
|
556
|
+
"and explicit execution timeouts to prevent denial-of-service."
|
|
557
|
+
)
|
|
558
|
+
elif v == AttackVector.STATE_INVARIANT.value:
|
|
559
|
+
directives.add(
|
|
560
|
+
"Verify state invariants: ensure operations are strictly idempotent where expected, "
|
|
561
|
+
"validate pre-conditions and post-conditions, and reject invalid state transitions."
|
|
562
|
+
)
|
|
563
|
+
else:
|
|
564
|
+
directives.add(f"Remediate failure in scenario `{f.scenario_id}`: {f.error_message or 'unhandled crash'}")
|
|
565
|
+
|
|
566
|
+
return sorted(directives)
|
|
567
|
+
|
|
568
|
+
def execute_swarm_attack(
|
|
569
|
+
self,
|
|
570
|
+
target_callable: Optional[Callable[..., Any]] = None,
|
|
571
|
+
scenarios: Optional[list[BreakScenario]] = None,
|
|
572
|
+
timeout_seconds: float = 3.0,
|
|
573
|
+
target_name: Optional[str] = None,
|
|
574
|
+
) -> RedTeamBreakageReport:
|
|
575
|
+
"""Executes the break scenarios against target_callable in-process.
|
|
576
|
+
|
|
577
|
+
Executes trusted callable objects in the current process. Source-code strings are rejected
|
|
578
|
+
and require a separate sandboxed executor for dynamic execution. Detects unhandled crashes,
|
|
579
|
+
memory/resource leaks, invariant violations, and timeouts.
|
|
580
|
+
"""
|
|
581
|
+
callable_fn = self._resolve_callable(target_callable)
|
|
582
|
+
actual_name = target_name or getattr(callable_fn, "__name__", "target")
|
|
583
|
+
|
|
584
|
+
if callable_fn is None:
|
|
585
|
+
report_id = f"redteam_{int(time.time())}_{uuid.uuid4().hex[:6]}"
|
|
586
|
+
finding = BreakFinding(
|
|
587
|
+
scenario_id="target_not_executable",
|
|
588
|
+
vector=AttackVector.CHAOS_ENVIRONMENT.value,
|
|
589
|
+
hypothesis="Target must be an executable Python Callable object",
|
|
590
|
+
broken=True,
|
|
591
|
+
error_message="TypeError: Source-code strings cannot be evaluated in-process for security reasons. Provide an executable Python Callable or use an isolated sandbox executor.",
|
|
592
|
+
severity="CRITICAL",
|
|
593
|
+
details={"target_executable": False},
|
|
594
|
+
)
|
|
595
|
+
return RedTeamBreakageReport(
|
|
596
|
+
report_id=report_id,
|
|
597
|
+
target_name=actual_name,
|
|
598
|
+
total_probes=1,
|
|
599
|
+
broken_count=1,
|
|
600
|
+
passed=False,
|
|
601
|
+
findings=[finding],
|
|
602
|
+
created_at=datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
603
|
+
remediation_directives=["Provide an executable Python callable or use an isolated sandbox executor."],
|
|
604
|
+
)
|
|
605
|
+
|
|
606
|
+
if not scenarios:
|
|
607
|
+
scenarios = self.generate_break_scenarios(target_name=actual_name)
|
|
608
|
+
|
|
609
|
+
findings: list[BreakFinding] = []
|
|
610
|
+
|
|
611
|
+
for sc in scenarios:
|
|
612
|
+
attack_fn = sc.attack_fn
|
|
613
|
+
repro_code = sc.metadata.get("reproduction") or f"{actual_name}(<adversarial_probe>)"
|
|
614
|
+
|
|
615
|
+
if attack_fn is None:
|
|
616
|
+
# If no attack function is defined, pass by default
|
|
617
|
+
findings.append(
|
|
618
|
+
BreakFinding(
|
|
619
|
+
scenario_id=sc.scenario_id,
|
|
620
|
+
vector=sc.vector.value if isinstance(sc.vector, AttackVector) else str(sc.vector),
|
|
621
|
+
hypothesis=sc.hypothesis,
|
|
622
|
+
broken=False,
|
|
623
|
+
severity="LOW",
|
|
624
|
+
details={"status": "no_attack_fn_provided"},
|
|
625
|
+
)
|
|
626
|
+
)
|
|
627
|
+
continue
|
|
628
|
+
|
|
629
|
+
# Execute attack with timeout containment
|
|
630
|
+
execution_result: dict[str, Any] = {"finished": False, "value": None, "error": None, "traceback": None}
|
|
631
|
+
|
|
632
|
+
def _runner() -> None:
|
|
633
|
+
try:
|
|
634
|
+
sig = inspect.signature(attack_fn)
|
|
635
|
+
if len(sig.parameters) > 0:
|
|
636
|
+
val = attack_fn(callable_fn)
|
|
637
|
+
else:
|
|
638
|
+
val = attack_fn()
|
|
639
|
+
execution_result["value"] = val
|
|
640
|
+
execution_result["finished"] = True
|
|
641
|
+
except Exception as exc:
|
|
642
|
+
execution_result["error"] = exc
|
|
643
|
+
execution_result["traceback"] = traceback.format_exc()
|
|
644
|
+
execution_result["finished"] = True
|
|
645
|
+
|
|
646
|
+
worker_thread = threading.Thread(target=_runner)
|
|
647
|
+
worker_thread.daemon = True
|
|
648
|
+
t0 = time.perf_counter()
|
|
649
|
+
worker_thread.start()
|
|
650
|
+
worker_thread.join(timeout=timeout_seconds)
|
|
651
|
+
duration_ms = (time.perf_counter() - t0) * 1000.0
|
|
652
|
+
|
|
653
|
+
vector_str = sc.vector.value if isinstance(sc.vector, AttackVector) else str(sc.vector)
|
|
654
|
+
|
|
655
|
+
if not execution_result["finished"]:
|
|
656
|
+
# Timed out!
|
|
657
|
+
findings.append(
|
|
658
|
+
BreakFinding(
|
|
659
|
+
scenario_id=sc.scenario_id,
|
|
660
|
+
vector=vector_str,
|
|
661
|
+
hypothesis=sc.hypothesis,
|
|
662
|
+
broken=True,
|
|
663
|
+
error_message=f"Execution timed out after {timeout_seconds:.1f}s (possible infinite loop or dead-lock)",
|
|
664
|
+
reproduction_code=repro_code,
|
|
665
|
+
severity="HIGH",
|
|
666
|
+
details={"duration_ms": duration_ms, "timed_out": True},
|
|
667
|
+
)
|
|
668
|
+
)
|
|
669
|
+
continue
|
|
670
|
+
|
|
671
|
+
if execution_result["error"] is not None:
|
|
672
|
+
exc = execution_result["error"]
|
|
673
|
+
# Determine if this exception represents a legitimate breakage or handled behavior.
|
|
674
|
+
# If callable crashed with unhandled raw crash:
|
|
675
|
+
is_unhandled_crash = isinstance(
|
|
676
|
+
exc,
|
|
677
|
+
(
|
|
678
|
+
AttributeError,
|
|
679
|
+
KeyError,
|
|
680
|
+
IndexError,
|
|
681
|
+
ZeroDivisionError,
|
|
682
|
+
UnboundLocalError,
|
|
683
|
+
RecursionError,
|
|
684
|
+
MemoryError,
|
|
685
|
+
AssertionError,
|
|
686
|
+
),
|
|
687
|
+
)
|
|
688
|
+
# If TypeError occurs because an attribute wasn't found or 'NoneType', it's unhandled crash
|
|
689
|
+
if isinstance(exc, TypeError) and any(kw in str(exc) for kw in ("NoneType", "subscriptable", "not callable")):
|
|
690
|
+
is_unhandled_crash = True
|
|
691
|
+
|
|
692
|
+
# If attack function specifically asserted resilience and failed:
|
|
693
|
+
severity = "CRITICAL" if isinstance(exc, (RecursionError, MemoryError, ZeroDivisionError)) else "HIGH"
|
|
694
|
+
|
|
695
|
+
findings.append(
|
|
696
|
+
BreakFinding(
|
|
697
|
+
scenario_id=sc.scenario_id,
|
|
698
|
+
vector=vector_str,
|
|
699
|
+
hypothesis=sc.hypothesis,
|
|
700
|
+
broken=True,
|
|
701
|
+
error_message=f"{type(exc).__name__}: {str(exc)}",
|
|
702
|
+
traceback_snippet=execution_result["traceback"],
|
|
703
|
+
reproduction_code=repro_code,
|
|
704
|
+
severity=severity,
|
|
705
|
+
details={"duration_ms": duration_ms, "exception_type": type(exc).__name__},
|
|
706
|
+
)
|
|
707
|
+
)
|
|
708
|
+
else:
|
|
709
|
+
# Returned normally - inspect return value for explicit breakage flags
|
|
710
|
+
val = execution_result["value"]
|
|
711
|
+
broken = False
|
|
712
|
+
error_msg: Optional[str] = None
|
|
713
|
+
|
|
714
|
+
if isinstance(val, dict):
|
|
715
|
+
if val.get("broken") is True or val.get("success") is False:
|
|
716
|
+
broken = True
|
|
717
|
+
error_msg = val.get("error") or val.get("error_message") or "Probe detected breakage condition"
|
|
718
|
+
elif isinstance(val, bool):
|
|
719
|
+
if sc.metadata.get("return_true_means_broken") is True and val is True:
|
|
720
|
+
broken = True
|
|
721
|
+
error_msg = "Attack function reported True (breakage detected)"
|
|
722
|
+
|
|
723
|
+
findings.append(
|
|
724
|
+
BreakFinding(
|
|
725
|
+
scenario_id=sc.scenario_id,
|
|
726
|
+
vector=vector_str,
|
|
727
|
+
hypothesis=sc.hypothesis,
|
|
728
|
+
broken=broken,
|
|
729
|
+
error_message=error_msg,
|
|
730
|
+
reproduction_code=repro_code if broken else None,
|
|
731
|
+
severity="MEDIUM" if broken else "LOW",
|
|
732
|
+
details={"duration_ms": duration_ms, "return_type": type(val).__name__},
|
|
733
|
+
)
|
|
734
|
+
)
|
|
735
|
+
|
|
736
|
+
broken_count = sum(1 for f in findings if f.broken)
|
|
737
|
+
passed = broken_count == 0
|
|
738
|
+
report_id = f"redteam_{int(time.time())}_{uuid.uuid4().hex[:6]}"
|
|
739
|
+
directives = self._derive_remediation_directives(findings)
|
|
740
|
+
|
|
741
|
+
return RedTeamBreakageReport(
|
|
742
|
+
report_id=report_id,
|
|
743
|
+
target_name=actual_name,
|
|
744
|
+
total_probes=len(findings),
|
|
745
|
+
broken_count=broken_count,
|
|
746
|
+
passed=passed,
|
|
747
|
+
findings=findings,
|
|
748
|
+
created_at=datetime.datetime.now(datetime.timezone.utc).isoformat(),
|
|
749
|
+
remediation_directives=directives,
|
|
750
|
+
)
|
|
751
|
+
|
|
752
|
+
def document_breakage(
|
|
753
|
+
self,
|
|
754
|
+
report: Union[RedTeamBreakageReport, dict[str, Any]],
|
|
755
|
+
output_path: Optional[str] = None,
|
|
756
|
+
) -> str:
|
|
757
|
+
"""Formats and optionally writes the markdown report to output_path."""
|
|
758
|
+
if isinstance(report, dict):
|
|
759
|
+
rep_obj = RedTeamBreakageReport.from_dict(report)
|
|
760
|
+
else:
|
|
761
|
+
rep_obj = report
|
|
762
|
+
|
|
763
|
+
markdown_content = rep_obj.to_markdown()
|
|
764
|
+
|
|
765
|
+
if output_path:
|
|
766
|
+
try:
|
|
767
|
+
parent = os.path.dirname(os.path.abspath(output_path))
|
|
768
|
+
os.makedirs(parent, exist_ok=True)
|
|
769
|
+
with open(output_path, "w", encoding="utf-8") as f:
|
|
770
|
+
f.write(markdown_content)
|
|
771
|
+
except Exception as e:
|
|
772
|
+
# Include warning if disk write fails
|
|
773
|
+
markdown_content += f"\n\n> [!WARNING]\n> Failed to persist markdown report to `{output_path}`: {e}"
|
|
774
|
+
|
|
775
|
+
return markdown_content
|
|
776
|
+
|
|
777
|
+
def verify_remediation(
|
|
778
|
+
self,
|
|
779
|
+
target_callable: Any,
|
|
780
|
+
prior_report: Union[RedTeamBreakageReport, dict[str, Any]],
|
|
781
|
+
additional_scenarios: Optional[list[BreakScenario]] = None,
|
|
782
|
+
auto_consolidate: bool = True,
|
|
783
|
+
timeout_seconds: float = 3.0,
|
|
784
|
+
) -> tuple[bool, RedTeamBreakageReport]:
|
|
785
|
+
"""Re-runs the scenarios that previously caused breakages against the remediated callable.
|
|
786
|
+
|
|
787
|
+
Includes optional additional regression scenarios.
|
|
788
|
+
Returns:
|
|
789
|
+
(all_fixed: bool, new_report: RedTeamBreakageReport)
|
|
790
|
+
"""
|
|
791
|
+
if isinstance(prior_report, dict):
|
|
792
|
+
p_rep = RedTeamBreakageReport.from_dict(prior_report)
|
|
793
|
+
else:
|
|
794
|
+
p_rep = prior_report
|
|
795
|
+
|
|
796
|
+
broken_ids = {f.scenario_id for f in p_rep.findings if f.broken}
|
|
797
|
+
|
|
798
|
+
# Generate fresh scenarios to get live attack functions
|
|
799
|
+
callable_fn = self._resolve_callable(target_callable)
|
|
800
|
+
target_name = getattr(callable_fn, "__name__", p_rep.target_name)
|
|
801
|
+
fresh_scenarios = self.generate_break_scenarios(target_name=target_name)
|
|
802
|
+
|
|
803
|
+
# Filter to previously broken scenarios
|
|
804
|
+
scenarios_to_rerun: list[BreakScenario] = [
|
|
805
|
+
s for s in fresh_scenarios if s.scenario_id in broken_ids or any(bid.endswith(s.scenario_id) for bid in broken_ids)
|
|
806
|
+
]
|
|
807
|
+
|
|
808
|
+
# If some broken IDs were custom or not in standard generator, reconstitute them from findings
|
|
809
|
+
matched_ids = {s.scenario_id for s in scenarios_to_rerun}
|
|
810
|
+
for f in p_rep.findings:
|
|
811
|
+
if f.broken and f.scenario_id not in matched_ids:
|
|
812
|
+
try:
|
|
813
|
+
vec = AttackVector(f.vector)
|
|
814
|
+
except ValueError:
|
|
815
|
+
vec = AttackVector.CHAOS_ENVIRONMENT
|
|
816
|
+
|
|
817
|
+
def _repro_probe(fn: Optional[Callable[..., Any]] = None, finding: BreakFinding = f) -> Any:
|
|
818
|
+
if fn is None:
|
|
819
|
+
raise TypeError("RedTeamSwarm target callable is missing or not executable")
|
|
820
|
+
sig = inspect.signature(fn)
|
|
821
|
+
if len(sig.parameters) == 0:
|
|
822
|
+
return fn()
|
|
823
|
+
return fn("remediation_verification_probe")
|
|
824
|
+
|
|
825
|
+
scenarios_to_rerun.append(
|
|
826
|
+
BreakScenario(
|
|
827
|
+
scenario_id=f.scenario_id,
|
|
828
|
+
vector=vec,
|
|
829
|
+
hypothesis=f.hypothesis,
|
|
830
|
+
expected_resilience="Remediated implementation must survive prior breaking probe",
|
|
831
|
+
attack_fn=_repro_probe,
|
|
832
|
+
metadata=f.details,
|
|
833
|
+
)
|
|
834
|
+
)
|
|
835
|
+
|
|
836
|
+
if additional_scenarios:
|
|
837
|
+
scenarios_to_rerun.extend(additional_scenarios)
|
|
838
|
+
|
|
839
|
+
# If no scenarios were previously broken and no additional, rerun all fresh scenarios as sanity check
|
|
840
|
+
if not scenarios_to_rerun:
|
|
841
|
+
scenarios_to_rerun = fresh_scenarios
|
|
842
|
+
|
|
843
|
+
new_report = self.execute_swarm_attack(
|
|
844
|
+
target_callable=callable_fn,
|
|
845
|
+
scenarios=scenarios_to_rerun,
|
|
846
|
+
timeout_seconds=timeout_seconds,
|
|
847
|
+
target_name=p_rep.target_name,
|
|
848
|
+
)
|
|
849
|
+
|
|
850
|
+
all_fixed = new_report.passed
|
|
851
|
+
if auto_consolidate and all_fixed and self.plasticity_engine is not None:
|
|
852
|
+
self.plasticity_engine.consolidate_task(
|
|
853
|
+
domain=p_rep.target_name,
|
|
854
|
+
task_id=p_rep.report_id,
|
|
855
|
+
broken_scenarios=[f.to_dict() for f in p_rep.findings if f.broken],
|
|
856
|
+
final_passed=True,
|
|
857
|
+
lessons=[
|
|
858
|
+
{
|
|
859
|
+
"trigger": f.hypothesis,
|
|
860
|
+
"mistake": f.error_message or "Unhandled break condition",
|
|
861
|
+
"defense": f.reproduction_code or "Hardened implementation",
|
|
862
|
+
"severity": f.severity,
|
|
863
|
+
}
|
|
864
|
+
for f in p_rep.findings
|
|
865
|
+
if f.broken
|
|
866
|
+
],
|
|
867
|
+
co_activated_nodes=["mutation", "test_harness", "red_team_swarm", "property_oracle"],
|
|
868
|
+
)
|
|
869
|
+
|
|
870
|
+
return all_fixed, new_report
|
|
871
|
+
|
|
872
|
+
def run_full_review_cycle(
|
|
873
|
+
self,
|
|
874
|
+
target_callable: Any,
|
|
875
|
+
target_name: str = "system",
|
|
876
|
+
custom_hypotheses: Optional[list[str]] = None,
|
|
877
|
+
auto_consolidate: bool = True,
|
|
878
|
+
) -> RedTeamBreakageReport:
|
|
879
|
+
"""Generates scenarios, executes swarm attack, and compiles the breakage report."""
|
|
880
|
+
callable_fn = self._resolve_callable(target_callable)
|
|
881
|
+
actual_name = getattr(callable_fn, "__name__", target_name)
|
|
882
|
+
effective_name = target_name if target_name and target_name != "system" else (actual_name or target_name)
|
|
883
|
+
scenarios = self.generate_break_scenarios(
|
|
884
|
+
target_name=effective_name,
|
|
885
|
+
custom_hypotheses=custom_hypotheses,
|
|
886
|
+
)
|
|
887
|
+
report = self.execute_swarm_attack(
|
|
888
|
+
target_callable=callable_fn,
|
|
889
|
+
scenarios=scenarios,
|
|
890
|
+
target_name=effective_name,
|
|
891
|
+
)
|
|
892
|
+
if auto_consolidate and self.plasticity_engine is not None:
|
|
893
|
+
if not report.passed:
|
|
894
|
+
self.plasticity_engine.consolidate_task(
|
|
895
|
+
domain=effective_name,
|
|
896
|
+
task_id=report.report_id,
|
|
897
|
+
broken_scenarios=[f.to_dict() for f in report.findings if f.broken],
|
|
898
|
+
final_passed=False,
|
|
899
|
+
co_activated_nodes=["test_harness", "red_team_swarm"],
|
|
900
|
+
)
|
|
901
|
+
else:
|
|
902
|
+
self.plasticity_engine.consolidate_task(
|
|
903
|
+
domain=effective_name,
|
|
904
|
+
task_id=report.report_id,
|
|
905
|
+
final_passed=True,
|
|
906
|
+
co_activated_nodes=["test_harness", "red_team_swarm", "diagnostics"],
|
|
907
|
+
)
|
|
908
|
+
return report
|