attackbenchlib 1.0.0a9__py3-none-any.whl

attackbench/__init__.py

@@ -0,0 +1,180 @@
+"""
+AttackBench - A Python package for benchmarking adversarial attacks.
+
+Usage:
+    import attackbench
+
+    results = attackbench.run_attack(model, dataset, attack, 'linf', device)
+    stats = attackbench.get_stats(results, 'linf')
+
+Optional subpackages:
+    - attacks: pip install attackbench[attacks]  (adversarial attack libraries)
+    - metrics: pip install attackbench[metrics]  (analysis & evaluation tools)
+    Both are optional and independent of each other.
+"""
+
+import importlib
+
+# Version info
+__version__ = "1.0.0"
+
+# ── Core API (always available) ──────────────────────────────────────────
+from .run import run_attack
+from .custom_components import create_custom_attack
+
+# ── Helpers to load objects (always available) ───────────────────────────
+from .datasets.registry import get_loader
+
+
+# ── RobustBench integration ─────────────────────────────────────────────
+def load_model(model_name, dataset='cifar10', threat_model='Linf', **kwargs):
+    """Load a RobustBench model and attach AttackBench metadata for automatic extraction."""
+    from robustbench import load_model as _rb_load_model
+    model = _rb_load_model(model_name=model_name, dataset=dataset, threat_model=threat_model, **kwargs)
+    model._attackbench_model = model_name
+    model._attackbench_dataset = dataset
+    return model
+
+# ── W&B integration ─────────────────────────────────────────────────────
+from .wandb import (
+    upload_precompiled_distances,
+    download_precompiled_distances,
+    list_available_distances,
+    upload_optimal_distances,
+    download_optimal_distances,
+    get_precompiled_distances,
+    get_optimal_distances,
+)
+
+# ── Lazy imports for optional subpackages (attacks/ and metrics/) ────────
+# These symbols are resolved on first access via __getattr__ (PEP 562).
+# If the subpackage is not installed, a clear ImportError is raised.
+
+_LAZY_ATTACKS = {
+    'get_attack':   ('attacks.registry', 'get_attack'),
+    'list_attacks': ('attacks.registry', 'list_attacks'),
+    'bomn_attack':  ('attacks.bomn',       'bomn_attack'),
+}
+
+_LAZY_MODELS = {
+    'get_model':    ('models.registry',    'get_model'),
+}
+
+_LAZY_METRICS = {
+    'get_stats':                   ('metrics.analysis',         'get_stats'),
+    'eval_optimality':             ('metrics.distances',        'eval_optimality'),
+    'ensemble_gain':               ('metrics.ensemble',         'ensemble_gain'),
+    'ensemble_distances':          ('metrics.ensemble',         'ensemble_distances'),
+    'compare_attacks':             ('metrics.analysis',         'compare_attacks'),
+    'compute_curves':              ('metrics.analysis',         'compute_curves'),
+    'compute_optimality':          ('metrics.analysis',         'compute_optimality'),
+    'compute_efficiency':          ('metrics.analysis',         'compute_efficiency'),
+    'compute_local_optimality':    ('metrics.optimality',       'compute_local_optimality'),
+    'compare_attacks_optimality':  ('metrics.optimality',       'compare_attacks_optimality'),
+    'compute_global_optimality':   ('metrics.global_optimality','compute_global_optimality'),
+    'create_attack_leaderboard':   ('metrics.global_optimality','create_attack_leaderboard'),
+    'compare_attacks_global':      ('metrics.global_optimality','compare_attacks_global'),
+    'format_leaderboard':          ('metrics.global_optimality','format_leaderboard'),
+}
+
+
+def __getattr__(name: str):
+    # ── Models (requires robustbench) ───────────────────────────────────
+    if name in _LAZY_MODELS:
+        submodule, attr = _LAZY_MODELS[name]
+        try:
+            mod = importlib.import_module(f'.{submodule}', __name__)
+            value = getattr(mod, attr)
+        except (ImportError, ModuleNotFoundError) as e:
+            raise ImportError(
+                f"attackbench.{name} requires robustbench. "
+                f"Install it with: pip install attackbench[models]"
+            ) from e
+        globals()[name] = value
+        return value
+
+    # ── Attacks (optional) ───────────────────────────────────────────────
+    if name in _LAZY_ATTACKS:
+        submodule, attr = _LAZY_ATTACKS[name]
+        try:
+            mod = importlib.import_module(f'.{submodule}', __name__)
+            value = getattr(mod, attr)
+        except (ImportError, ModuleNotFoundError) as e:
+            raise ImportError(
+                f"attackbench.{name} requires the 'attacks' subpackage. "
+                f"Install it with: pip install attackbench[attacks]"
+            ) from e
+        globals()[name] = value          # cache for subsequent accesses
+        return value
+
+    # ── Metrics (optional) ───────────────────────────────────────────────
+    if name in _LAZY_METRICS:
+        submodule, attr = _LAZY_METRICS[name]
+        try:
+            mod = importlib.import_module(f'.{submodule}', __name__)
+            value = getattr(mod, attr)
+        except (ImportError, ModuleNotFoundError) as e:
+            raise ImportError(
+                f"attackbench.{name} requires the 'metrics' subpackage. "
+                f"Install it with: pip install attackbench[metrics]"
+            ) from e
+        globals()[name] = value
+        return value
+
+    raise AttributeError(f"module 'attackbench' has no attribute '{name}'")
+
+
+def __dir__():
+    """Support tab-completion for lazy attributes."""
+    public = list(globals().keys())
+    public.extend(_LAZY_MODELS.keys())
+    public.extend(_LAZY_ATTACKS.keys())
+    public.extend(_LAZY_METRICS.keys())
+    return public
+
+
+__all__ = [
+    'run_attack',
+    'get_stats',
+
+    # Helpers to load objects
+    'load_model',
+    'get_model',
+    'get_loader',
+    'get_attack',
+    'list_attacks',
+
+    # Custom components
+    'create_custom_attack',
+
+    # BoMN composite attack
+    'bomn_attack',
+
+    # Analysis functions
+    'eval_optimality',
+    'ensemble_gain',
+    'ensemble_distances',
+    'compare_attacks',
+    'compute_curves',
+    'compute_optimality',
+    'compute_efficiency',
+
+    # Stage 3: Local Optimality
+    'compute_local_optimality',
+    'compare_attacks_optimality',
+
+    # Stage 4-5: Global Optimality & Ranking
+    'compute_global_optimality',
+    'create_attack_leaderboard',
+    'compare_attacks_global',
+    'format_leaderboard',
+
+    # W&B functions
+    'upload_precompiled_distances',
+    'download_precompiled_distances',
+    'list_available_distances',
+    'get_precompiled_distances',
+    'upload_optimal_distances',
+    'download_optimal_distances',
+    'get_optimal_distances',
+]

attackbench/adv_lib_sub.py

@@ -0,0 +1,239 @@
+"""
+adv_lib_sub.py - Substitute for adv_lib external dependency.
+
+This module provides internal implementations of functions that were
+previously imported from adv_lib, allowing AttackBench to work without
+the external adv_lib dependency.
+
+Contains:
+- Distance metrics (l0, l1, l2, linf)
+- Default metrics dictionary
+- Loss functions (difference_of_logits)
+- Model utilities (normalize_model, NormalizeLayer)
+"""
+
+from collections import OrderedDict
+from typing import Optional, Tuple, Union
+
+import torch
+from torch import nn, Tensor
+
+
+# =============================================================================
+# DISTANCE METRICS
+# =============================================================================
+
+def l0_distances(x: Tensor, x_adv: Tensor, dim: Optional[int] = None) -> Tensor:
+    """
+    Compute L0 distance (number of perturbed elements) between original and adversarial samples.
+    
+    Args:
+        x: Original inputs, shape (batch_size, ...)
+        x_adv: Adversarial inputs, shape (batch_size, ...)
+        dim: Starting dimension from which to flatten and compute distance.
+             If None, flattens from dim 1.
+        
+    Returns:
+        L0 distances per sample
+    """
+    diff = (x - x_adv).abs()
+    if dim is not None:
+        return (diff.flatten(start_dim=dim) > 1e-10).sum(dim=-1).float()
+    else:
+        return (diff.flatten(start_dim=1) > 1e-10).sum(dim=1).float()
+
+
+def l1_distances(x: Tensor, x_adv: Tensor, dim: Optional[int] = None) -> Tensor:
+    """
+    Compute L1 distance (sum of absolute differences) between original and adversarial samples.
+    
+    Args:
+        x: Original inputs, shape (batch_size, ...)
+        x_adv: Adversarial inputs, shape (batch_size, ...)
+        dim: Starting dimension from which to flatten and compute distance.
+             If None, flattens from dim 1.
+        
+    Returns:
+        L1 distances per sample
+    """
+    diff = (x - x_adv).abs()
+    if dim is not None:
+        return diff.flatten(start_dim=dim).sum(dim=-1)
+    else:
+        return diff.flatten(start_dim=1).sum(dim=1)
+
+
+def l2_distances(x: Tensor, x_adv: Tensor, dim: Optional[int] = None) -> Tensor:
+    """
+    Compute L2 (Euclidean) distance between original and adversarial samples.
+    
+    Args:
+        x: Original inputs, shape (batch_size, ...)
+        x_adv: Adversarial inputs, shape (batch_size, ...)
+        dim: Starting dimension from which to flatten and compute distance.
+             If None, flattens from dim 1.
+        
+    Returns:
+        L2 distances per sample
+    """
+    diff = x - x_adv
+    if dim is not None:
+        return diff.flatten(start_dim=dim).norm(p=2, dim=-1)
+    else:
+        return diff.flatten(start_dim=1).norm(p=2, dim=1)
+
+
+def linf_distances(x: Tensor, x_adv: Tensor, dim: Optional[int] = None) -> Tensor:
+    """
+    Compute L∞ distance (maximum absolute difference) between original and adversarial samples.
+    
+    Args:
+        x: Original inputs, shape (batch_size, ...)
+        x_adv: Adversarial inputs, shape (batch_size, ...)
+        dim: Starting dimension from which to flatten and compute distance.
+             If None, flattens from dim 1.
+        
+    Returns:
+        L∞ distances per sample
+    """
+    diff = (x - x_adv).abs()
+    if dim is not None:
+        return diff.flatten(start_dim=dim).max(dim=-1)[0]
+    else:
+        return diff.flatten(start_dim=1).max(dim=1)[0]
+
+
+# =============================================================================
+# DEFAULT METRICS
+# =============================================================================
+
+# Default metrics dictionary for tracking distance metrics
+# This matches the adv_lib._default_metrics structure
+_default_metrics = OrderedDict([
+    ('linf', linf_distances),
+    ('l2', l2_distances),
+    ('l1', l1_distances),
+    ('l0', l0_distances),
+])
+
+
+# =============================================================================
+# LOSS FUNCTIONS
+# =============================================================================
+
+def difference_of_logits(logits: Tensor, labels: Tensor, targeted: bool = False) -> Tensor:
+    """
+    Compute the Difference of Logits (DL) loss.
+    
+    DL loss is defined as:
+    - For untargeted: logit[true_class] - max(logit[other_classes])
+    - For targeted: max(logit[other_classes]) - logit[target_class]
+    
+    Positive values indicate misclassification (successful attack).
+    
+    Args:
+        logits: Model output logits, shape (batch_size, num_classes)
+        labels: True labels (untargeted) or target labels (targeted), shape (batch_size,)
+        targeted: Whether this is a targeted attack
+        
+    Returns:
+        DL loss per sample, shape (batch_size,)
+    """
+    batch_size, num_classes = logits.shape
+    
+    # Get logit values for the target/true class
+    target_logits = logits.gather(1, labels.unsqueeze(1)).squeeze(1)
+    
+    # Create a mask for other classes
+    mask = torch.ones_like(logits).scatter_(1, labels.unsqueeze(1), 0.0)
+    
+    # Get maximum logit among other classes
+    other_logits = (logits * mask + (1 - mask) * float('-inf')).max(dim=1)[0]
+    
+    if targeted:
+        # For targeted: want other_logits > target_logits (positive when successful)
+        return other_logits - target_logits
+    else:
+        # For untargeted: want target_logits < other_logits (positive when successful)
+        return target_logits - other_logits
+
+
+# =============================================================================
+# MODEL UTILITIES
+# =============================================================================
+
+class NormalizeLayer(nn.Module):
+    """Normalization layer to be prepended to a model."""
+    
+    def __init__(self, mean: Union[Tuple[float, ...], Tensor], 
+                 std: Union[Tuple[float, ...], Tensor]):
+        """
+        Initialize normalization layer.
+        
+        Args:
+            mean: Mean values for each channel
+            std: Standard deviation values for each channel
+        """
+        super(NormalizeLayer, self).__init__()
+        
+        if isinstance(mean, (tuple, list)):
+            mean = torch.tensor(mean)
+        if isinstance(std, (tuple, list)):
+            std = torch.tensor(std)
+            
+        self.register_buffer('mean', mean.view(1, -1, 1, 1))
+        self.register_buffer('std', std.view(1, -1, 1, 1))
+    
+    def forward(self, x: Tensor) -> Tensor:
+        """Normalize input tensor."""
+        return (x - self.mean) / self.std
+
+
+def normalize_model(model: nn.Module, 
+                    mean: Union[Tuple[float, ...], Tensor],
+                    std: Union[Tuple[float, ...], Tensor]) -> nn.Module:
+    """
+    Prepend a normalization layer to a model.
+    
+    Creates a sequential model with normalization as the first layer,
+    allowing the model to accept [0, 1] normalized inputs.
+    
+    Args:
+        model: PyTorch model to wrap
+        mean: Mean values for normalization (per channel)
+        std: Standard deviation values for normalization (per channel)
+        
+    Returns:
+        Sequential model with normalization prepended
+        
+    Example:
+        >>> model = resnet18()
+        >>> # Normalize with ImageNet stats
+        >>> normalized_model = normalize_model(
+        ...     model, 
+        ...     mean=(0.485, 0.456, 0.406),
+        ...     std=(0.229, 0.224, 0.225)
+        ... )
+    """
+    normalize_layer = NormalizeLayer(mean, std)
+    return nn.Sequential(normalize_layer, model)
+
+
+# =============================================================================
+# EXPORTS
+# =============================================================================
+
+__all__ = [
+    # Distance metrics
+    'l0_distances',
+    'l1_distances', 
+    'l2_distances',
+    'linf_distances',
+    # Default metrics
+    '_default_metrics',
+    # Loss functions
+    'difference_of_logits',
+    # Model utilities
+    'NormalizeLayer',
+    'normalize_model',
+]

attackbench/attacks/README.md

@@ -0,0 +1,65 @@
+# Adding an attack
+
+To add an attack that can be called through the sacred CLI, you need to implement two functions.
+
+### Config function
+
+The first function will be the named config and should follow the template:
+
+```python
+def <library prefix>_<config name>():
+    name = '<name of the attack>'
+    source = '<name of the library>'
+    threat_model = '<name of the threat model>'
+    option1 = 0.01
+```
+
+The library prefix corresponds to a shorter version of the library name; for instance, Foolbox's prefix is `fb`.
+All other variables of the config function will correspond to the attack's option.
+This function should be placed in the `<library>/configs.py` file.
+
+### Getter function
+
+The second function to implemented is the getter function, which will return the attack as a callable. This getter
+function should follow the template:
+
+```python
+def get_<library prefix>_<attack name>(option1: float) -> Callable:
+    return ...
+```
+
+The `<attack name>` in the name of the getter function should match exactly the name of the attack in the config
+function: `name = <attack name>`. This is necessary to determine which getter function to call when calling a named
+config.
+
+### Example
+
+In `foolbox/configs.py`, the DDN attack is added with the two functions:
+
+```python
+def fb_ddn():
+    name = 'ddn'
+    source = 'foolbox'
+    threat_model = 'l2'
+    init_epsilon = 1
+    num_steps = 100
+    gamma = 0.05
+
+
+def get_fb_ddn(init_epsilon: float, num_steps: int, gamma: float) -> Callable:
+    return partial(DDNAttack, init_epsilon=init_epsilon, steps=num_steps, gamma=gamma)
+```
+
+Additionally, one could add a second named config for DDN by simply implementing a second config function:
+
+```python
+def fb_ddn_large_gamma():
+    name = 'ddn'
+    source = 'foolbox'
+    threat_model = 'l2'
+    init_epsilon = 1
+    num_steps = 100
+    gamma = 0.5
+```
+
+This second named config would point to the same getter function.

attackbench/attacks/__init__.py

@@ -0,0 +1,17 @@
+minimal_search_steps = 20
+minimal_init_eps = {
+    'l0': 100,
+    'l1': 10,
+    'l2': 1,
+    'linf': 1 / 255,
+}
+
+# Pre-configured attack instances — importable as `from attackbench.attacks import pgd`
+from ..preconfigured import (
+    pgd, fgsm, apgd, fab, fmn, deepfool, superdeepfool, trust_region
+)
+
+__all__ = [
+    'minimal_search_steps', 'minimal_init_eps',
+    'pgd', 'fgsm', 'apgd', 'fab', 'fmn', 'deepfool', 'superdeepfool', 'trust_region',
+]