adv-lib 0.2.2__py3-none-any.whl

adv_lib/attacks/segmentation/primal_dual_gradient_descent.py

@@ -0,0 +1,349 @@
+import math
+from functools import partial
+from typing import Optional
+
+import torch
+from torch import Tensor, nn
+from torch.autograd import grad
+from torch.nn import functional as F
+
+from adv_lib.attacks.primal_dual_gradient_descent import l0_proximal_, l1_proximal, l23_proximal, l2_proximal_, \
+    linf_proximal_
+from adv_lib.distances.lp_norms import l0_distances, l1_distances, l2_distances, linf_distances
+from adv_lib.utils.losses import difference_of_logits
+from adv_lib.utils.visdom_logger import VisdomLogger
+
+
+def softmax_plus_one(tensor: torch.Tensor) -> torch.Tensor:
+    zero_pad = F.pad(tensor.flatten(1), pad=(1, 0), mode='constant', value=0)
+    return zero_pad.softmax(dim=1)[:, 1:].view_as(tensor)
+
+
+def pdgd(model: nn.Module,
+         inputs: Tensor,
+         labels: Tensor,
+         masks: Tensor = None,
+         targeted: bool = False,
+         adv_threshold: float = 0.99,
+         num_steps: int = 500,
+         random_init: float = 0,
+         primal_lr: float = 0.1,
+         primal_lr_decrease: float = 0.01,
+         dual_ratio_init: float = 1,
+         dual_lr: float = 0.1,
+         dual_lr_decrease: float = 0.1,
+         dual_ema: float = 0.9,
+         dual_min_ratio: float = 1e-6,
+         constraint_masking: bool = False,
+         mask_decay: bool = False,
+         callback: Optional[VisdomLogger] = None) -> Tensor:
+    """Primal-Dual Gradient Descent (PDGD) attack from https://arxiv.org/abs/2106.01538 adapted to semantic
+    segmentation. This version is only suitable for the L2-norm."""
+    attack_name = 'PDGD L2'
+    device = inputs.device
+    batch_size = len(inputs)
+    batch_view = lambda tensor: tensor.view(batch_size, *[1] * (inputs.ndim - 1))
+    multiplier = -1 if targeted else 1
+    log_min_dual_ratio = math.log(dual_min_ratio)
+
+    # Setup variables
+    r = torch.zeros_like(inputs, requires_grad=True)
+    if random_init:
+        nn.init.uniform_(r, -random_init, random_init)
+        r.data.add_(inputs).clamp_(min=0, max=1).sub_(inputs)
+
+    # Adam variables
+    exp_avg = torch.zeros_like(inputs)
+    exp_avg_sq = torch.zeros_like(inputs)
+    β_1, β_2 = 0.9, 0.999
+
+    # dual variables
+    λ = torch.zeros_like(labels, dtype=torch.double)
+
+    # Init trackers
+    best_l2 = torch.full((batch_size,), float('inf'), device=device)
+    best_adv = inputs.clone()
+    adv_found = torch.zeros_like(best_l2, dtype=torch.bool)
+    best_adv_percent = torch.zeros_like(best_l2)
+
+    for i in range(num_steps):
+
+        adv_inputs = inputs + r
+        logits = model(adv_inputs)
+        l2 = r.flatten(1).norm(p=2, dim=1)
+
+        if i == 0:
+            num_classes = logits.size(1)
+            if masks is None:
+                masks = labels < num_classes
+            masks_sum = masks.flatten(1).sum(dim=1)
+            labels_ = labels.clone()
+            labels_[~masks] = 0
+
+            masks_inf = torch.zeros_like(masks, dtype=torch.float).masked_fill_(~masks, float('inf'))
+            labels_infhot = torch.zeros_like(logits.detach()).scatter(1, labels_.unsqueeze(1), float('inf'))
+            dl_func = partial(difference_of_logits, labels=labels_, labels_infhot=labels_infhot)
+
+            # init dual variables with masks
+            λ.add_(masks_sum.float().mul_(dual_ratio_init).log_().neg_().view(-1, 1, 1))
+            λ[~masks] = -float('inf')
+            λ_ema = softmax_plus_one(λ)
+
+            # init constraint masking
+            k = ((1 - adv_threshold) * masks_sum).long()  # number of constraints that can be violated
+            constraint_mask = masks
+            constraint_inf_mask = torch.zeros_like(constraint_mask, dtype=torch.float)
+            constraint_inf_mask.masked_fill_(~constraint_mask, float('inf'))
+
+        # track progress
+        pred = logits.argmax(dim=1)
+        pixel_is_adv = (pred == labels) if targeted else (pred != labels)
+        adv_percent = (pixel_is_adv & masks).flatten(1).sum(dim=1) / masks_sum
+        is_adv = adv_percent >= adv_threshold
+        is_smaller = l2 <= best_l2
+        improves_constraints = adv_percent >= best_adv_percent.clamp_max(adv_threshold)
+        is_better_adv = (is_smaller & is_adv) | (~adv_found & improves_constraints)
+        adv_found.logical_or_(is_adv)
+        best_l2 = torch.where(is_better_adv, l2.detach(), best_l2)
+        best_adv_percent = torch.where(is_better_adv, adv_percent, best_adv_percent)
+        best_adv = torch.where(batch_view(is_better_adv), adv_inputs.detach(), best_adv)
+
+        m_y = multiplier * dl_func(logits)
+
+        if constraint_masking:
+            if mask_decay:
+                k = ((1 - adv_threshold) * masks_sum).mul_(i / (num_steps - 1)).long()
+            if k.any():
+                top_constraints = m_y.detach().sub(masks_inf).flatten(1).topk(k=k.max()).values
+                ξ = top_constraints.gather(1, k.unsqueeze(1) - 1).squeeze(1)
+                constraint_mask = masks & (m_y <= ξ.view(-1, 1, 1))
+                constraint_inf_mask.fill_(0).masked_fill_(~constraint_mask, float('inf'))
+
+        if i:
+            λ_ema.lerp_(softmax_plus_one(λ - constraint_inf_mask), weight=1 - dual_ema)
+        λ_ema_masked = λ_ema * constraint_mask
+        λ_1 = 1 - λ_ema_masked.flatten(1).sum(dim=1)
+
+        L_r = λ_1 * l2 + F.softplus(m_y).mul(λ_ema_masked).flatten(1).sum(dim=1)
+
+        grad_r = grad(L_r.sum(), inputs=r, only_inputs=True)[0]
+        grad_λ = m_y.detach().sign().mul_(masks)
+
+        # Adam algorithm
+        exp_avg.lerp_(grad_r, weight=1 - β_1)
+        exp_avg_sq.mul_(β_2).addcmul_(grad_r, grad_r, value=1 - β_2)
+        bias_correction1 = 1 - β_1 ** (i + 1)
+        bias_correction2 = 1 - β_2 ** (i + 1)
+        denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(1e-8)
+        # primal step size exponential decay
+        step_size = primal_lr * primal_lr_decrease ** (i / num_steps)
+        # gradient descent on primal variables
+        r.data.addcdiv_(exp_avg, denom, value=-step_size / bias_correction1)
+
+        # projection on feasible set
+        r.data.add_(inputs).clamp_(min=0, max=1).sub_(inputs)
+
+        # gradient ascent on dual variables and exponential moving average
+        θ_λ = dual_lr * ((num_steps - 1 - i) / (num_steps - 1) * (1 - dual_lr_decrease) + dual_lr_decrease)
+        λ.add_(grad_λ, alpha=θ_λ).clamp_(min=log_min_dual_ratio, max=-log_min_dual_ratio)
+        λ[~masks] = -float('inf')
+
+        if callback is not None:
+            callback.accumulate_line('m_y', i, m_y.mean(), title=f'{attack_name} - Logit difference')
+            callback.accumulate_line('1 - sum(λ)', i, λ_1.mean(), title=f'{attack_name} - Dual variables')
+            callback.accumulate_line(['θ_r', 'θ_λ'], i, [step_size, θ_λ], title=f'{attack_name} - Learning rates')
+            callback.accumulate_line(['l2', 'best_l2'], i, [l2.mean(), best_l2.mean()],
+                                     title=f'{attack_name} - L2 norms')
+            callback.accumulate_line(['adv%', 'best_adv%'], i, [adv_percent.mean(), best_adv_percent.mean()],
+                                     title=f'{attack_name} - APSRs')
+
+            if (i + 1) % (num_steps // 20) == 0 or (i + 1) == num_steps:
+                callback.update_lines()
+
+    return best_adv
+
+
+def pdpgd(model: nn.Module,
+          inputs: Tensor,
+          labels: Tensor,
+          norm: float,
+          masks: Tensor = None,
+          targeted: bool = False,
+          adv_threshold: float = 0.99,
+          num_steps: int = 500,
+          random_init: float = 0,
+          proximal_operator: Optional[float] = None,
+          primal_lr: float = 0.1,
+          primal_lr_decrease: float = 0.01,
+          dual_ratio_init: float = 1,
+          dual_lr: float = 0.1,
+          dual_lr_decrease: float = 0.1,
+          dual_ema: float = 0.9,
+          dual_min_ratio: float = 1e-12,
+          proximal_steps: int = 5,
+          ε_threshold: float = 1e-2,
+          constraint_masking: bool = False,
+          mask_decay: bool = False,
+          callback: Optional[VisdomLogger] = None) -> Tensor:
+    """Primal-Dual Proximal Gradient Descent (PDPGD) attacks from https://arxiv.org/abs/2106.01538 adapted to semantic
+    segmentation."""
+    attack_name = f'PDPGD L{norm}'
+    _distance = {
+        0: l0_distances,
+        1: l1_distances,
+        2: l2_distances,
+        float('inf'): linf_distances,
+    }
+    _proximal_operator = {
+        0: l0_proximal_,
+        1: l1_proximal,
+        2: l2_proximal_,
+        float('inf'): linf_proximal_,
+        23: l23_proximal,
+    }
+    device = inputs.device
+    batch_size = len(inputs)
+    batch_view = lambda tensor: tensor.view(batch_size, *[1] * (inputs.ndim - 1))
+    multiplier = -1 if targeted else 1
+    distance = _distance[norm]
+    proximity_operator = _proximal_operator[norm if proximal_operator is None else proximal_operator]
+    log_min_dual_ratio = math.log(dual_min_ratio)
+
+    # Setup variables
+    r = torch.zeros_like(inputs, requires_grad=True)
+    if random_init:
+        nn.init.uniform_(r, -random_init, random_init)
+        r.data.add_(inputs).clamp_(min=0, max=1).sub_(inputs)
+
+    # Adam variables
+    exp_avg = torch.zeros_like(inputs)
+    exp_avg_sq = torch.zeros_like(inputs)
+    β_1, β_2 = 0.9, 0.999
+
+    # dual variables
+    λ = torch.zeros_like(labels, dtype=torch.double)
+
+    # Init trackers
+    best_dist = torch.full((batch_size,), float('inf'), device=device)
+    best_adv = inputs.clone()
+    adv_found = torch.zeros_like(best_dist, dtype=torch.bool)
+    best_adv_percent = torch.zeros_like(best_dist)
+
+    for i in range(num_steps):
+
+        adv_inputs = inputs + r
+        logits = model(adv_inputs)
+        dist = distance(adv_inputs.detach(), inputs)
+
+        if i == 0:
+            num_classes = logits.size(1)
+            if masks is None:
+                masks = labels < num_classes
+            masks_sum = masks.flatten(1).sum(dim=1)
+            labels_ = labels.clone()
+            labels_[~masks] = 0
+
+            masks_inf = torch.zeros_like(masks, dtype=torch.float).masked_fill_(~masks, float('inf'))
+            labels_infhot = torch.zeros_like(logits.detach()).scatter(1, labels_.unsqueeze(1), float('inf'))
+            dl_func = partial(difference_of_logits, labels=labels_, labels_infhot=labels_infhot)
+
+            # init dual variables with masks
+            λ.add_(masks_sum.float().mul_(dual_ratio_init).log_().neg_().view(-1, 1, 1))
+            λ[~masks] = -float('inf')
+            λ_ema = softmax_plus_one(λ)
+
+            # init constraint masking
+            k = ((1 - adv_threshold) * masks_sum).long()  # number of constraints that can be violated
+            constraint_mask = masks
+            constraint_inf_mask = torch.zeros_like(constraint_mask, dtype=torch.float)
+            constraint_inf_mask.masked_fill_(~constraint_mask, float('inf'))
+
+        # track progress
+        pred = logits.argmax(dim=1)
+        pixel_is_adv = (pred == labels) if targeted else (pred != labels)
+        adv_percent = (pixel_is_adv & masks).flatten(1).sum(dim=1) / masks_sum
+        is_adv = adv_percent >= adv_threshold
+        is_smaller = dist <= best_dist
+        improves_constraints = adv_percent >= best_adv_percent.clamp_max(adv_threshold)
+        is_better_adv = (is_smaller & is_adv) | (~adv_found & improves_constraints)
+        adv_found.logical_or_(is_adv)
+        best_dist = torch.where(is_better_adv, dist.detach(), best_dist)
+        best_adv_percent = torch.where(is_better_adv, adv_percent, best_adv_percent)
+        best_adv = torch.where(batch_view(is_better_adv), adv_inputs.detach(), best_adv)
+
+        m_y = multiplier * dl_func(logits)
+
+        if constraint_masking:
+            if mask_decay:
+                k = ((1 - adv_threshold) * masks_sum).mul_(i / (num_steps - 1)).long()
+            if k.any():
+                top_constraints = m_y.detach().sub(masks_inf).flatten(1).topk(k=k.max()).values
+                ξ = top_constraints.gather(1, k.unsqueeze(1) - 1).squeeze(1)
+                constraint_mask = masks & (m_y <= ξ.view(-1, 1, 1))
+                constraint_inf_mask.fill_(0).masked_fill_(~constraint_mask, float('inf'))
+
+        if i:
+            λ_ema.lerp_(softmax_plus_one(λ - constraint_inf_mask), weight=1 - dual_ema)
+        λ_ema_masked = λ_ema * constraint_mask
+
+        cls_loss = F.softplus(m_y).mul(λ_ema_masked).flatten(1).sum(dim=1)
+
+        grad_r = grad(cls_loss.sum(), inputs=r, only_inputs=True)[0]
+        grad_λ = m_y.detach().sign().mul_(constraint_mask)
+
+        # Adam algorithm
+        exp_avg.lerp_(grad_r, weight=1 - β_1)
+        exp_avg_sq.mul_(β_2).addcmul_(grad_r, grad_r, value=1 - β_2)
+        bias_correction1 = 1 - β_1 ** (i + 1)
+        bias_correction2 = 1 - β_2 ** (i + 1)
+        denom = (exp_avg_sq.sqrt() / math.sqrt(bias_correction2)).add_(1e-8)
+        # primal step size exponential decay
+        step_size = primal_lr * primal_lr_decrease ** (i / num_steps)
+        # gradient descent on primal variables
+        r.data.addcdiv_(exp_avg, denom, value=-step_size / bias_correction1)
+
+        # projection on feasible set
+        r.data.add_(inputs).clamp_(min=0, max=1).sub_(inputs)
+
+        # proximal adam https://arxiv.org/abs/1910.10094
+        ψ_max = denom.flatten(1).amax(dim=1)
+        effective_lr = step_size / ψ_max
+
+        # proximal sub-iterations variables
+        z_curr = r.detach()
+        ε = torch.ones_like(best_dist)
+        λ_sum = λ_ema_masked.flatten(1).sum(dim=1)
+        μ = ((1 - λ_sum) / λ_sum).to(dtype=torch.float).mul_(effective_lr)
+        H_div = denom / batch_view(ψ_max)
+        for _ in range(proximal_steps):
+            z_prev = z_curr
+
+            z_new = proximity_operator(z_curr.addcmul(H_div, z_curr - r.detach(), value=-1), batch_view(μ))
+            z_new.add_(inputs).clamp_(min=0, max=1).sub_(inputs)
+
+            z_curr = torch.where(batch_view(ε > ε_threshold), z_new, z_prev)
+            ε = torch.norm((z_curr - z_prev).flatten(1), p=2, dim=1, out=ε).div_(z_curr.flatten(1).norm(p=2, dim=1))
+
+            if (ε < ε_threshold).all():
+                break
+
+        r.data = z_curr
+
+        # gradient ascent on dual variables and exponential moving average
+        θ_λ = dual_lr * ((num_steps - 1 - i) / (num_steps - 1) * (1 - dual_lr_decrease) + dual_lr_decrease)
+        λ.add_(grad_λ, alpha=θ_λ).clamp_(min=log_min_dual_ratio, max=-log_min_dual_ratio)
+        λ[~masks] = -float('inf')
+
+        if callback is not None:
+            callback.accumulate_line('m_y', i, m_y.mean(), title=f'{attack_name} - Logit difference')
+            callback.accumulate_line('1 - sum(λ)', i, (1 - λ_sum).mean(), title=f'{attack_name} - Dual variables')
+            callback.accumulate_line(['θ_r', 'θ_λ'], i, [step_size, θ_λ], title=f'{attack_name} - Learning rates')
+            callback.accumulate_line([f'l{norm}', f'best_l{norm}'], i, [dist.mean(), best_dist.mean()],
+                                     title=f'{attack_name} - L{norm} norms')
+            callback.accumulate_line(['adv%', 'best_adv%'], i, [adv_percent.mean(), best_adv_percent.mean()],
+                                     title=f'{attack_name} - APSRs')
+
+            if (i + 1) % (num_steps // 20) == 0 or (i + 1) == num_steps:
+                callback.update_lines()
+
+    return best_adv

adv_lib/attacks/self_adaptive_norm_update.py

@@ -0,0 +1,127 @@
+import math
+from typing import Optional
+
+import torch
+from torch import nn, Tensor
+from torch.autograd import grad
+from torch.nn import functional as F
+
+from adv_lib.utils.visdom_logger import VisdomLogger
+
+
+def anu(model: nn.Module,
+        inputs: Tensor,
+        labels: Tensor,
+        targeted: bool = False,
+        steps: int = 100,
+        γ_init: float = 0.05,
+        α_γ: float = 0.05,
+        init_norm: float = 1.,
+        levels: Optional[int] = 256,
+        callback: Optional[VisdomLogger] = None) -> Tensor:
+    """
+    Self Adaptive Norm Update (ANU) attack from https://www.scitepress.org/PublishedPapers/2021/101861/101861.pdf.
+
+    Parameters
+    ----------
+    model : nn.Module
+        Model to attack.
+    inputs : Tensor
+        Inputs to attack. Should be in [0, 1].
+    labels : Tensor
+        Labels corresponding to the inputs if untargeted, else target labels.
+    targeted : bool
+        Whether to perform a targeted attack or not.
+    steps : int
+        Number of optimization steps.
+    γ : float
+        Factor by which the norm will be modified. new_norm = norm * (1 + or - γ).
+    init_norm : float
+        Initial value for the norm of the attack.
+    levels : int
+        If not None, the returned adversarials will have quantized values to the specified number of levels.
+
+    adaptive : bool
+        If True, adapt the value of γ based on the last two iterations as in "Self-adaptive Norm Update for Faster
+        Gradient-based L2 Adversarial Attacks and Defenses" (https://www.scitepress.org/Papers/2021/101861/).
+    α_γ : float
+        Factor by which γ will be modified: new_γ = γ * (1 + or - α_γ). An addition clipping has been added (not
+        mentioned in the original paper) to ensure that 1 - γ remains strictly positive.
+
+    callback : Optional
+
+    Returns
+    -------
+    adv_inputs : Tensor
+        Modified inputs to be adversarial to the model.
+
+    """
+    if inputs.min() < 0 or inputs.max() > 1: raise ValueError('Input values should be in the [0, 1] range.')
+    device = inputs.device
+    batch_size = len(inputs)
+    batch_view = lambda tensor: tensor.view(batch_size, *[1] * (inputs.ndim - 1))
+
+    # Init variables
+    multiplier = -1 if targeted else 1
+    δ = torch.zeros_like(inputs, requires_grad=True)
+    γ = torch.full((batch_size,), γ_init, device=device, dtype=torch.float)
+    ε = torch.full_like(γ, init_norm)
+    worst_norm = torch.max(inputs, 1 - inputs).flatten(1).norm(p=2, dim=1)
+
+    # Init trackers
+    best_l2 = worst_norm.clone()
+    best_δ = torch.zeros_like(inputs)
+    adv_found = torch.zeros(batch_size, dtype=torch.bool, device=device)
+    is_adv_prev = adv_found.clone()
+
+    for i in range(steps):
+        α = torch.tensor(0.01 + (1 - 0.01) * (1 + math.cos(math.pi * i / steps)) / 2, device=device)
+
+        l2 = δ.data.flatten(1).norm(p=2, dim=1)
+        adv_inputs = inputs + δ
+        logits = model(adv_inputs)
+        pred_labels = logits.argmax(1)
+        ce_loss = F.cross_entropy(logits, labels, reduction='none')
+        loss = multiplier * ce_loss
+
+        is_adv = (pred_labels == labels) if targeted else (pred_labels != labels)
+        is_smaller = l2 < best_l2
+        is_both = is_adv & is_smaller
+        adv_found.logical_or_(is_adv)
+        best_l2 = torch.where(is_both, l2, best_l2)
+        best_δ = torch.where(batch_view(is_both), δ.detach(), best_δ)
+
+        δ_grad = grad(loss.sum(), δ, only_inputs=True)[0]
+        # renorming gradient
+        grad_norms = δ_grad.flatten(1).norm(p=2, dim=1)
+        δ_grad.div_(batch_view(grad_norms))
+        # avoid nan or inf if gradient is 0
+        if (zero_grad := (grad_norms < 1e-12)).any():
+            δ_grad[zero_grad] = torch.randn_like(δ_grad[zero_grad])
+
+        if callback is not None:
+            cosine = F.cosine_similarity(δ_grad.flatten(1), δ.data.flatten(1), dim=1).mean()
+            callback.accumulate_line('ce', i, ce_loss.mean())
+            callback_best = best_l2.masked_select(adv_found).mean()
+            callback.accumulate_line(['ε', 'γ', 'l2', 'best_l2'], i, [ε.mean(), γ.mean(), l2.mean(), callback_best])
+            callback.accumulate_line(['cosine', 'α', 'success'], i, [cosine, α, adv_found.float().mean()])
+
+            if (i + 1) % (steps // 20) == 0 or (i + 1) == steps:
+                callback.update_lines()
+
+        # gradient step
+        δ.data.add_(δ_grad, alpha=α)
+
+        γ = torch.where(is_adv == is_adv_prev, (1 + α_γ) * γ, (1 - α_γ) * γ).clamp_(min=0, max=1)
+        ε = torch.where(is_adv, (1 - γ) * ε, (1 + γ) * ε)
+        ε = torch.minimum(ε, worst_norm)
+
+        δ.data.mul_(batch_view(ε / δ.data.flatten(1).norm(p=2, dim=1)))
+        δ.data.add_(inputs).clamp_(0, 1)
+        if levels is not None:
+            δ.data.mul_(levels - 1).round_().div_(levels - 1)
+        δ.data.sub_(inputs)
+
+        is_adv_prev = is_adv
+
+    return inputs + best_δ

adv_lib/attacks/sigma_zero.py

@@ -0,0 +1,119 @@
+# Adapted from https://github.com/Cinofix/sigma-zero-adversarial-attack
+import math
+import warnings
+
+import torch
+from torch import Tensor, nn
+from torch.autograd import grad
+
+from adv_lib.utils.losses import difference_of_logits
+
+
+def sigma_zero(model: nn.Module,
+               inputs: Tensor,
+               labels: Tensor,
+               num_steps: int = 1000,
+               η_0: float = 1.0,
+               σ: float = 0.001,
+               τ_0: float = 0.3,
+               τ_factor: float = 0.01,
+               grad_norm: float = float('inf'),
+               targeted: bool = False) -> Tensor:
+    """
+    σ-zero attack from https://arxiv.org/abs/2402.01879.
+
+    Parameters
+    ----------
+    model : nn.Module
+        Model to attack.
+    inputs : Tensor
+        Inputs to attack. Should be in [0, 1].
+    labels : Tensor
+        Labels corresponding to the inputs if untargeted, else target labels.
+    num_steps : int
+        Number of optimization steps. Corresponds to the number of forward and backward propagations.
+    η_0 : float
+        Initial step size.
+    σ : float
+        \ell_0 approximation parameter: smaller values produce sharper approximations while larger values produce a
+        smoother approximation.
+    τ_0 : float
+        Initial sparsity threshold.
+    τ_factor : float
+        Threshold adjustment factor w.r.t. step size η.
+    grad_norm: float
+        Norm to use for gradient normalization.
+    targeted : bool
+        Attack is untargeted only: will raise a warning and return inputs if targeted is True.
+
+    Returns
+    -------
+    best_adv : Tensor
+        Perturbed inputs (inputs + perturbation) that are adversarial and have smallest distance with the original
+        inputs.
+
+    """
+    if targeted:
+        warnings.warn('σ-zero attack is untargeted only. Returning inputs.')
+        return inputs
+
+    batch_size, numel = len(inputs), inputs[0].numel()
+    batch_view = lambda tensor: tensor.view(batch_size, *[1] * (inputs.ndim - 1))
+
+    δ = torch.zeros_like(inputs, requires_grad=True)
+    # Adam variables
+    exp_avg = torch.zeros_like(inputs)
+    exp_avg_sq = torch.zeros_like(inputs)
+    β_1, β_2 = 0.9, 0.999
+
+    best_l0 = inputs.new_full((batch_size,), numel)
+    best_adv = inputs.clone()
+    τ = torch.full_like(best_l0, τ_0)
+
+    η = η_0
+    for i in range(num_steps):
+        adv_inputs = inputs + δ
+
+        # compute loss
+        logits = model(adv_inputs)
+        dl_loss = difference_of_logits(logits, labels).clamp_(min=0)
+        δ_square = δ.square()
+        l0_approx_normalized = (δ_square / (δ_square + σ)).flatten(1).mean(dim=1)
+
+        # keep best solutions
+        predicted_classes = logits.argmax(dim=1)
+        l0_norm = δ.data.flatten(1).norm(p=0, dim=1)
+        is_adv = (predicted_classes == labels) if targeted else (predicted_classes != labels)
+        is_smaller = l0_norm < best_l0
+        is_both = is_adv & is_smaller
+        best_l0 = torch.where(is_both, l0_norm, best_l0)
+        best_adv = torch.where(batch_view(is_both), adv_inputs.detach(), best_adv)
+
+        # compute loss and gradient
+        adv_loss = (dl_loss + l0_approx_normalized).sum()
+        δ_grad = grad(adv_loss, inputs=δ, only_inputs=True)[0]
+
+        # normalize gradient based on grad_norm type
+        δ_inf_norm = δ_grad.flatten(1).norm(p=grad_norm, dim=1).clamp_(min=1e-12)
+        δ_grad.div_(batch_view(δ_inf_norm))
+
+        # adam computations
+        exp_avg.lerp_(δ_grad, weight=1 - β_1)
+        exp_avg_sq.mul_(β_2).addcmul_(δ_grad, δ_grad, value=1 - β_2)
+        bias_correction1 = 1 - β_1 ** (i + 1)
+        bias_correction2 = 1 - β_2 ** (i + 1)
+        denom = exp_avg_sq.sqrt().div_(bias_correction2 ** 0.5).add_(1e-8)
+
+        # step and clamp
+        δ.data.addcdiv_(exp_avg, denom, value=-η / bias_correction1)
+        δ.data.add_(inputs).clamp_(min=0, max=1).sub_(inputs)
+
+        # update step size with cosine annealing
+        η = 0.1 * η_0 + 0.9 * η_0 * (1 + math.cos(math.pi * i / num_steps)) / 2
+        # dynamic thresholding
+        τ.add_(torch.where(is_adv, τ_factor * η, -τ_factor * η)).clamp_(min=0, max=1)
+
+        # filter components
+        δ.data[δ.data.abs() < batch_view(τ)] = 0
+
+    return best_adv