swarm-test 0.1.0__py3-none-any.whl

swarm_test/__init__.py

@@ -0,0 +1,43 @@
+"""
+swarm-test: The first reliability testing framework for multi-agent AI systems.
+
+Quick start::
+
+    from swarm_test import SwarmProbe
+
+    probe = SwarmProbe(crew)
+    report = probe.run_all()
+    report.print_summary()
+"""
+
+from swarm_test.core.models import (
+    AgentNode,
+    EventType,
+    Finding,
+    InteractionEvent,
+    Severity,
+    SwarmReport,
+    TestResult,
+    TestStatus,
+)
+from swarm_test.core.graph import SwarmGraph
+from swarm_test.core.probe import SwarmProbe
+
+__version__ = "0.1.0"
+__author__ = "swarm-test contributors"
+__license__ = "MIT"
+
+__all__ = [
+    # Main API
+    "SwarmProbe",
+    "SwarmGraph",
+    # Models
+    "AgentNode",
+    "InteractionEvent",
+    "Finding",
+    "TestResult",
+    "SwarmReport",
+    "Severity",
+    "EventType",
+    "TestStatus",
+]

swarm_test/attacks/__init__.py

@@ -0,0 +1 @@
+"""Attack modules for swarm-test chaos testing."""

swarm_test/attacks/base.py

@@ -0,0 +1,38 @@
+"""Base class for all swarm chaos attacks."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import TYPE_CHECKING
+
+from swarm_test.core.models import TestResult
+
+if TYPE_CHECKING:
+    from swarm_test.core.graph import SwarmGraph
+
+
+class BaseAttack(ABC):
+    """
+    Abstract base class that all attack modules must implement.
+
+    Subclasses should override ``name``, ``description``, and ``run()``.
+    """
+
+    name: str = "base_attack"
+    description: str = "Base chaos attack."
+
+    @abstractmethod
+    def run(self, graph: SwarmGraph) -> TestResult:
+        """
+        Execute the attack against the provided SwarmGraph.
+
+        Args:
+            graph: A ``SwarmGraph`` instance containing agent nodes and events.
+
+        Returns:
+            A ``TestResult`` with findings, metrics, and status.
+        """
+        ...
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(name={self.name!r})"

swarm_test/attacks/blast_radius.py

@@ -0,0 +1,201 @@
+"""Blast Radius Attack — quantify impact of targeted agent failures."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List, Set
+
+import networkx as nx
+
+from swarm_test.attacks.base import BaseAttack
+from swarm_test.core.models import Finding, Severity, TestResult, TestStatus
+
+logger = logging.getLogger(__name__)
+
+
+class BlastRadiusAttack(BaseAttack):
+    """
+    Performs a systematic blast radius analysis across the entire swarm:
+    - Identifies agents whose failure would most impact the system
+    - Detects single points of failure (articulation points)
+    - Quantifies the critical path and its length
+    - Checks for adequate redundancy
+
+    This is a quantitative complement to CascadeFailureAttack,
+    focusing on topology-level metrics rather than simulation.
+    """
+
+    name = "blast_radius"
+    description = (
+        "Topological blast radius analysis: identifies critical agents, "
+        "single points of failure, and lack of redundancy in the swarm graph."
+    )
+
+    def run(self, graph: Any) -> TestResult:
+        findings: List[Finding] = []
+        metrics: Dict[str, Any] = {
+            "total_agents": 0,
+            "total_edges": 0,
+            "single_points_of_failure": [],
+            "critical_path": [],
+            "critical_path_length": 0,
+            "top_blast_agents": [],
+            "redundancy_score": 0.0,
+            "graph_density": 0.0,
+        }
+
+        g = graph.graph
+        n = g.number_of_nodes()
+        e = g.number_of_edges()
+        metrics["total_agents"] = n
+        metrics["total_edges"] = e
+
+        if n < 2:
+            return TestResult(
+                test_name=self.name,
+                status=TestStatus.PASSED,
+                findings=[],
+                metrics={"note": "Need ≥2 agents for blast radius analysis"},
+            )
+
+        # Graph density
+        density = nx.density(g)
+        metrics["graph_density"] = round(density, 4)
+
+        # 1. Single Points of Failure (articulation points)
+        spofs = graph.find_single_points_of_failure()
+        metrics["single_points_of_failure"] = [
+            g.nodes[s].get("name", s) for s in spofs if s in g
+        ]
+
+        for spof_id in spofs:
+            if spof_id not in g:
+                continue
+            spof_name = g.nodes[spof_id].get("name", spof_id)
+            blast = graph.get_blast_radius(spof_id)
+            findings.append(
+                Finding(
+                    test_name=self.name,
+                    severity=Severity.CRITICAL,
+                    title=f"Single Point of Failure: {spof_name}",
+                    description=(
+                        f"Agent '{spof_name}' is an articulation point — removing it "
+                        f"would disconnect the agent communication graph. "
+                        f"Blast radius: {blast['impact_percentage']:.1f}% of agents affected."
+                    ),
+                    affected_agents=[spof_id] + blast["downstream_agents"],
+                    evidence={
+                        "agent_id": spof_id,
+                        "impact_percentage": blast["impact_percentage"],
+                        "downstream_count": len(blast["downstream_agents"]),
+                    },
+                    remediation=(
+                        "Introduce redundant agents or fallback paths. "
+                        "Consider load balancing across multiple agents for this role. "
+                        "Implement circuit breakers on connections to this agent."
+                    ),
+                )
+            )
+
+        # 2. Critical path analysis
+        critical_path = graph.get_critical_path()
+        metrics["critical_path"] = [
+            g.nodes[n].get("name", n) for n in critical_path if n in g
+        ]
+        metrics["critical_path_length"] = len(critical_path)
+
+        if len(critical_path) >= 4:
+            path_names = metrics["critical_path"]
+            findings.append(
+                Finding(
+                    test_name=self.name,
+                    severity=Severity.HIGH,
+                    title=f"Long critical path: {len(critical_path)} agents",
+                    description=(
+                        f"The critical path spans {len(critical_path)} agents: "
+                        f"{' → '.join(path_names)}. "
+                        "Failure anywhere on this path creates a service outage."
+                    ),
+                    affected_agents=critical_path,
+                    evidence={"path": critical_path, "path_names": path_names},
+                    remediation=(
+                        "Shorten the critical path by parallelizing independent agents. "
+                        "Add checkpointing and recovery mechanisms along this path."
+                    ),
+                )
+            )
+
+        # 3. Top blast radius agents
+        blast_scores = []
+        for node in g.nodes():
+            blast = graph.get_blast_radius(node)
+            blast_scores.append(
+                {
+                    "agent_id": node,
+                    "agent_name": g.nodes[node].get("name", node),
+                    "impact_pct": blast["impact_percentage"],
+                    "downstream_count": len(blast["downstream_agents"]),
+                }
+            )
+        blast_scores.sort(key=lambda x: x["impact_pct"], reverse=True)
+        metrics["top_blast_agents"] = blast_scores[:5]
+
+        # 4. Redundancy score — ratio of edges to minimum spanning tree edges
+        # Higher = more redundant paths
+        undirected = g.to_undirected()
+        if undirected.number_of_edges() > 0 and nx.is_connected(undirected):
+            mst_edges = undirected.number_of_nodes() - 1
+            actual_edges = undirected.number_of_edges()
+            redundancy = (actual_edges - mst_edges) / max(mst_edges, 1)
+            metrics["redundancy_score"] = round(redundancy, 3)
+
+            if redundancy < 0.1 and n > 3:
+                findings.append(
+                    Finding(
+                        test_name=self.name,
+                        severity=Severity.MEDIUM,
+                        title=f"Low redundancy score: {redundancy:.2f}",
+                        description=(
+                            f"The swarm graph has a redundancy score of {redundancy:.2f} "
+                            "(close to a tree structure with no alternative paths). "
+                            "A single edge failure could create an unreachable agent."
+                        ),
+                        affected_agents=list(g.nodes()),
+                        evidence={"redundancy_score": redundancy, "edge_count": actual_edges},
+                        remediation=(
+                            "Add fallback communication paths between agents. "
+                            "Implement retry logic with alternative agent routing."
+                        ),
+                    )
+                )
+
+        # 5. Isolated agents (zero in-degree AND zero out-degree, excluding root)
+        isolated = [
+            n for n in g.nodes()
+            if g.in_degree(n) == 0 and g.out_degree(n) == 0
+        ]
+        if isolated:
+            isolated_names = [g.nodes[i].get("name", i) for i in isolated]
+            findings.append(
+                Finding(
+                    test_name=self.name,
+                    severity=Severity.LOW,
+                    title=f"{len(isolated)} isolated agent(s) detected",
+                    description=(
+                        f"Agents {isolated_names} have no connections to any other agent. "
+                        "They will never be tested under load and may represent dead code."
+                    ),
+                    affected_agents=isolated,
+                    evidence={"isolated_agents": isolated_names},
+                    remediation=(
+                        "Remove unused agents or integrate them into the swarm workflow."
+                    ),
+                )
+            )
+
+        return TestResult(
+            test_name=self.name,
+            status=TestStatus.PASSED,
+            findings=findings,
+            metrics=metrics,
+        )

swarm_test/attacks/cascade.py

@@ -0,0 +1,120 @@
+"""Cascade Failure Attack — simulate agent failure propagation."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any, Dict, List
+
+from swarm_test.attacks.base import BaseAttack
+from swarm_test.core.models import Finding, Severity, TestResult, TestStatus
+
+logger = logging.getLogger(__name__)
+
+
+class CascadeFailureAttack(BaseAttack):
+    """
+    Simulates a cascade failure by disabling each agent one at a time and
+    measuring how many downstream agents would be impacted.
+
+    Findings are raised when:
+    - A single agent failure affects >50% of the swarm (CRITICAL)
+    - A single agent failure affects >25% (HIGH)
+    - A single agent failure affects >10% (MEDIUM)
+    """
+
+    name = "cascade_failure"
+    description = (
+        "Simulates agent failures and measures downstream propagation "
+        "to detect dangerous cascade paths."
+    )
+
+    THRESHOLDS = [
+        (50.0, Severity.CRITICAL, "Catastrophic cascade potential"),
+        (25.0, Severity.HIGH, "High cascade risk"),
+        (10.0, Severity.MEDIUM, "Moderate cascade risk"),
+    ]
+
+    def run(self, graph: Any) -> TestResult:
+        findings: List[Finding] = []
+        metrics: Dict[str, Any] = {
+            "agents_tested": 0,
+            "max_impact_pct": 0.0,
+            "most_critical_agent": None,
+            "cascade_paths": [],
+        }
+
+        nodes = list(graph.graph.nodes())
+        metrics["agents_tested"] = len(nodes)
+
+        if len(nodes) < 2:
+            return TestResult(
+                test_name=self.name,
+                status=TestStatus.PASSED,
+                findings=[],
+                metrics={"note": "Need ≥2 agents for cascade analysis"},
+            )
+
+        worst_impact = 0.0
+        worst_agent = None
+
+        for agent_id in nodes:
+            blast = graph.get_blast_radius(agent_id)
+            impact_pct = blast["impact_percentage"]
+
+            if impact_pct > worst_impact:
+                worst_impact = impact_pct
+                worst_agent = agent_id
+
+            downstream = blast["downstream_agents"]
+            if downstream:
+                metrics["cascade_paths"].append(
+                    {
+                        "agent": blast["agent_name"],
+                        "downstream_count": len(downstream),
+                        "impact_pct": impact_pct,
+                    }
+                )
+
+            for threshold, severity, label in self.THRESHOLDS:
+                if impact_pct >= threshold:
+                    agent_name = blast["agent_name"]
+                    findings.append(
+                        Finding(
+                            test_name=self.name,
+                            severity=severity,
+                            title=f"{label}: {agent_name} failure cascades to {len(downstream)} agents",
+                            description=(
+                                f"Agent '{agent_name}' (id={agent_id}) has a blast radius of "
+                                f"{impact_pct:.1f}% — failure would directly or indirectly "
+                                f"impact {len(downstream)} of {blast['total_agents']} agents."
+                            ),
+                            affected_agents=[agent_id] + downstream,
+                            evidence=blast,
+                            remediation=(
+                                "Introduce circuit breakers, health checks, and fallback agents "
+                                "to isolate failures. Consider replicating this agent."
+                            ),
+                        )
+                    )
+                    break  # Only report the highest severity per agent
+
+        # Deduplicate findings (same agent may match multiple thresholds due to loop)
+        # We break after first match, so no dedup needed.
+
+        metrics["max_impact_pct"] = round(worst_impact, 2)
+        metrics["most_critical_agent"] = (
+            graph.graph.nodes[worst_agent].get("name", worst_agent)
+            if worst_agent and worst_agent in graph.graph
+            else None
+        )
+
+        # Sort cascade paths by impact descending
+        metrics["cascade_paths"].sort(key=lambda x: x["impact_pct"], reverse=True)
+        metrics["cascade_paths"] = metrics["cascade_paths"][:10]  # Top 10
+
+        return TestResult(
+            test_name=self.name,
+            status=TestStatus.PASSED,  # overridden by probe based on findings
+            findings=findings,
+            metrics=metrics,
+        )

swarm_test/attacks/collusion.py

@@ -0,0 +1,247 @@
+"""Collusion Detection Attack — identify coordinated agent misbehaviour."""
+
+from __future__ import annotations
+
+import logging
+from collections import defaultdict
+from typing import Any, Dict, List, Set, Tuple
+
+import networkx as nx
+
+from swarm_test.attacks.base import BaseAttack
+from swarm_test.core.models import EventType, Finding, Severity, TestResult, TestStatus
+
+logger = logging.getLogger(__name__)
+
+
+class CollusionDetectionAttack(BaseAttack):
+    """
+    Detects potential collusion between agents by analyzing:
+    1. Dense bi-directional communication clusters (cliques)
+    2. Coordinated error suppression (agents consistently hiding failures)
+    3. Echo chambers — groups of agents exclusively talking to each other
+    4. Cyclic dependency groups that bypass orchestrator oversight
+    """
+
+    name = "collusion_detection"
+    description = (
+        "Identifies clusters of agents with unusually dense or cyclic communication "
+        "patterns that could indicate coordinated misbehaviour or oversight bypass."
+    )
+
+    # A clique of this size or larger is flagged
+    MIN_CLIQUE_SIZE = 3
+    # Fraction of bidirectional edges within a group to be considered an echo chamber
+    ECHO_CHAMBER_THRESHOLD = 0.8
+
+    def run(self, graph: Any) -> TestResult:
+        findings: List[Finding] = []
+        metrics: Dict[str, Any] = {
+            "cliques_found": 0,
+            "echo_chambers": 0,
+            "cyclic_groups": 0,
+            "suppressed_error_pairs": 0,
+        }
+
+        g = graph.graph
+        if g.number_of_nodes() < 3:
+            return TestResult(
+                test_name=self.name,
+                status=TestStatus.PASSED,
+                findings=[],
+                metrics={"note": "Need ≥3 agents for collusion analysis"},
+            )
+
+        # 1. Dense clique detection on undirected projection
+        undirected = g.to_undirected(as_view=True)
+        cliques = [c for c in nx.find_cliques(undirected) if len(c) >= self.MIN_CLIQUE_SIZE]
+        metrics["cliques_found"] = len(cliques)
+
+        for clique in cliques:
+            agent_names = [g.nodes[n].get("name", n) for n in clique]
+            findings.append(
+                Finding(
+                    test_name=self.name,
+                    severity=Severity.HIGH,
+                    title=f"Dense communication clique detected ({len(clique)} agents)",
+                    description=(
+                        f"Agents {agent_names} form a fully-connected communication clique. "
+                        f"This dense subgraph may indicate coordinated behaviour "
+                        f"or information sharing outside the orchestrator's oversight."
+                    ),
+                    affected_agents=clique,
+                    evidence={"clique": clique, "agent_names": agent_names},
+                    remediation=(
+                        "Audit communication logs for this agent cluster. "
+                        "Enforce hub-and-spoke topology via an orchestrator agent. "
+                        "Add rate limiting on peer-to-peer agent communication."
+                    ),
+                )
+            )
+
+        # 2. Echo chamber detection — groups exclusively talking among themselves
+        echo_findings = self._detect_echo_chambers(graph)
+        metrics["echo_chambers"] = len(echo_findings)
+        findings.extend(echo_findings)
+
+        # 3. Cyclic dependency groups that loop without orchestrator
+        cycle_findings = self._detect_collusion_cycles(graph)
+        metrics["cyclic_groups"] = len(cycle_findings)
+        findings.extend(cycle_findings)
+
+        # 4. Coordinated error suppression
+        suppression_findings = self._detect_error_suppression(graph)
+        metrics["suppressed_error_pairs"] = len(suppression_findings)
+        findings.extend(suppression_findings)
+
+        return TestResult(
+            test_name=self.name,
+            status=TestStatus.PASSED,
+            findings=findings,
+            metrics=metrics,
+        )
+
+    def _detect_echo_chambers(self, graph: Any) -> List[Finding]:
+        """Detect strongly connected components that form isolated echo chambers."""
+        findings = []
+        g = graph.graph
+
+        # SCCs with ≥3 members where internal edge density is high
+        sccs = [scc for scc in nx.strongly_connected_components(g) if len(scc) >= 3]
+
+        for scc in sccs:
+            subgraph = g.subgraph(scc)
+            internal_edges = subgraph.number_of_edges()
+            possible_edges = len(scc) * (len(scc) - 1)  # directed
+            density = internal_edges / possible_edges if possible_edges > 0 else 0
+
+            # Check if agents in SCC communicate much more with each other than outside
+            external_edges = sum(
+                1
+                for src, dst in g.edges()
+                if (src in scc) != (dst in scc)  # XOR: one in, one out
+            )
+            total_edges = g.number_of_edges()
+            isolation_ratio = 1 - (external_edges / max(total_edges, 1))
+
+            if density >= self.ECHO_CHAMBER_THRESHOLD and isolation_ratio > 0.6:
+                agent_names = [g.nodes[n].get("name", n) for n in scc]
+                findings.append(
+                    Finding(
+                        test_name=self.name,
+                        severity=Severity.MEDIUM,
+                        title=f"Echo chamber: {len(scc)} agents with {density:.0%} internal density",
+                        description=(
+                            f"Agents {agent_names} form an echo chamber: "
+                            f"{density:.0%} internal communication density, "
+                            f"{isolation_ratio:.0%} isolation from the rest of the swarm."
+                        ),
+                        affected_agents=list(scc),
+                        evidence={
+                            "density": round(density, 3),
+                            "isolation_ratio": round(isolation_ratio, 3),
+                            "internal_edges": internal_edges,
+                            "external_edges": external_edges,
+                        },
+                        remediation=(
+                            "Break up isolated agent clusters. Require all agent "
+                            "sub-groups to report through a central orchestrator."
+                        ),
+                    )
+                )
+        return findings
+
+    @staticmethod
+    def _detect_collusion_cycles(graph: Any) -> List[Finding]:
+        """Flag cycles that bypass any orchestrator/manager node."""
+        findings = []
+        g = graph.graph
+
+        # Identify potential orchestrator nodes (high in-degree + out-degree)
+        orchestrators: Set[str] = set()
+        for node in g.nodes():
+            role = g.nodes[node].get("role", "").lower()
+            if any(r in role for r in ("manager", "orchestrator", "coordinator", "planner")):
+                orchestrators.add(node)
+
+        if not orchestrators:
+            return []  # Can't determine bypass without known orchestrators
+
+        cycles = list(nx.simple_cycles(g))
+        for cycle in cycles:
+            # Check if none of the orchestrators are in this cycle
+            if not any(o in cycle for o in orchestrators):
+                agent_names = [g.nodes[n].get("name", n) for n in cycle]
+                findings.append(
+                    Finding(
+                        test_name="collusion_detection",
+                        severity=Severity.HIGH,
+                        title=f"Orchestrator-bypass cycle: {' → '.join(agent_names)}",
+                        description=(
+                            f"Agents {agent_names} form a cyclic dependency that "
+                            "completely bypasses any orchestrator or manager agent. "
+                            "This allows coordinated actions without oversight."
+                        ),
+                        affected_agents=cycle,
+                        evidence={"cycle": cycle, "orchestrators": list(orchestrators)},
+                        remediation=(
+                            "Redesign the agent graph so all cycles pass through "
+                            "an orchestrator node that can audit decisions."
+                        ),
+                    )
+                )
+        return findings
+
+    @staticmethod
+    def _detect_error_suppression(graph: Any) -> List[Finding]:
+        """
+        Detect pairs of agents where one consistently sends failed events
+        but the other's subsequent outputs show success (error suppression).
+        """
+        findings = []
+        events = graph.events
+
+        # Group events by (src, dst) edge
+        edge_events: Dict[Tuple[str, str], List[Any]] = defaultdict(list)
+        for event in events:
+            edge_events[(event.source_agent_id, event.target_agent_id)].append(event)
+
+        for (src, dst), evts in edge_events.items():
+            total = len(evts)
+            if total < 3:
+                continue
+            failures = [e for e in evts if not e.success]
+            failure_rate = len(failures) / total
+
+            if failure_rate > 0.5:
+                # Check if downstream agent's events all appear successful
+                downstream_events = [
+                    e for e in events if e.source_agent_id == dst and e.success
+                ]
+                if downstream_events and len(downstream_events) >= len(failures):
+                    src_name = graph.graph.nodes.get(src, {}).get("name", src) if src in graph.graph else src
+                    dst_name = graph.graph.nodes.get(dst, {}).get("name", dst) if dst in graph.graph else dst
+                    findings.append(
+                        Finding(
+                            test_name="collusion_detection",
+                            severity=Severity.MEDIUM,
+                            title=f"Possible error suppression: {src_name} → {dst_name}",
+                            description=(
+                                f"Edge {src_name}→{dst_name} has a {failure_rate:.0%} failure rate "
+                                f"({len(failures)}/{total} events failed), yet downstream agent "
+                                f"'{dst_name}' continues reporting success. "
+                                "This may indicate coordinated error suppression."
+                            ),
+                            affected_agents=[src, dst],
+                            evidence={
+                                "failure_rate": round(failure_rate, 3),
+                                "failed_events": len(failures),
+                                "total_events": total,
+                            },
+                            remediation=(
+                                "Implement independent health monitoring for each agent. "
+                                "Require agents to propagate failure signals up the chain."
+                            ),
+                        )
+                    )
+        return findings