homa 0.1.94__tar.gz → 0.2.0__tar.gz

{homa-0.1.94 → homa-0.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: homa
-Version: 0.1.94
+Version: 0.2.0
 Summary: A curated list of machine learning and deep learning helpers.
 Author-email: Taha Shieenavaz <tahashieenavaz@gmail.com>
 Requires-Python: >=3.7

{homa-0.1.94 → homa-0.2.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "homa"
-version = "0.1.94"
+version = "0.2.0"
 description = "A curated list of machine learning and deep learning helpers."
 authors = [
   { name="Taha Shieenavaz", email="tahashieenavaz@gmail.com" },

homa-0.2.0/src/homa/activations/classes/APLU.py

@@ -0,0 +1,86 @@
+import torch
+from torch import nn
+from torch.nn.parameter import Parameter, UninitializedParameter
+import torch.nn.functional as F
+
+
+class APLU(nn.Module):
+    def __init__(self, max_input: float = 1.0):
+        super().__init__()
+        self.max_input = float(max_input)
+        self.alpha = UninitializedParameter()
+        self.beta = UninitializedParameter()
+        self.gamma = UninitializedParameter()
+        self.xi = UninitializedParameter()
+        self.psi = UninitializedParameter()
+        self.mu = UninitializedParameter()
+        self._num_channels = None
+
+    def _initialize_parameters(self, x: torch.Tensor):
+        if x.ndim < 2:
+            raise ValueError(
+                f"Input tensor must have at least 2 dimensions (N, C), but got shape {tuple(x.shape)}"
+            )
+
+        channels = int(x.shape[1])
+        self._num_channels = channels
+        param_shape = [1] * x.ndim
+        param_shape[1] = channels
+
+        with torch.no_grad():
+            self.alpha = Parameter(
+                torch.zeros(param_shape, dtype=x.dtype, device=x.device)
+            )
+            self.beta = Parameter(
+                torch.zeros(param_shape, dtype=x.dtype, device=x.device)
+            )
+            self.gamma = Parameter(
+                torch.zeros(param_shape, dtype=x.dtype, device=x.device)
+            )
+            self.xi = Parameter(
+                torch.empty(param_shape, dtype=x.dtype, device=x.device).uniform_(
+                    0.0, self.max_input
+                )
+            )
+            self.psi = Parameter(
+                torch.empty(param_shape, dtype=x.dtype, device=x.device).uniform_(
+                    0.0, self.max_input
+                )
+            )
+            self.mu = Parameter(
+                torch.empty(param_shape, dtype=x.dtype, device=x.device).uniform_(
+                    0.0, self.max_input
+                )
+            )
+
+    def reset_parameters(self):
+        if isinstance(self.alpha, UninitializedParameter):
+            return
+
+        with torch.no_grad():
+            self.alpha.zero_()
+            self.beta.zero_()
+            self.gamma.zero_()
+            self.xi.uniform_(0.0, self.max_input)
+            self.psi.uniform_(0.0, self.max_input)
+            self.mu.uniform_(0.0, self.max_input)
+
+    def forward(self, x: torch.Tensor):
+        if isinstance(self.alpha, UninitializedParameter):
+            self._initialize_parameters(x)
+
+        if x.ndim < 2:
+            raise ValueError(
+                f"Input tensor must have at least 2 dimensions (N, C), but got shape {tuple(x.shape)}"
+            )
+        if self._num_channels is not None and x.shape[1] != self._num_channels:
+            raise RuntimeError(
+                f"APLU was initialized with C={self._num_channels} but got C={x.shape[1]}. "
+                "Create a new APLU for a different channel size."
+            )
+
+        a = F.relu(x)
+        b = self.alpha * F.relu(-x + self.xi)
+        c = self.beta * F.relu(-x + self.psi)
+        d = self.gamma * F.relu(-x + self.mu)
+        return a + b + c + d

homa-0.2.0/src/homa/activations/classes/GALU.py

@@ -0,0 +1,67 @@
+import torch
+from torch import nn
+from torch.nn.parameter import Parameter, UninitializedParameter
+import torch.nn.functional as F
+
+
+class GALU(nn.Module):
+    def __init__(self, max_input: float = 1.0):
+        super().__init__()
+        if max_input <= 0:
+            raise ValueError("max_input must be positive.")
+        self.max_input = float(max_input)
+        self.alpha: torch.Tensor = UninitializedParameter()
+        self.beta: torch.Tensor = UninitializedParameter()
+        self.gamma: torch.Tensor = UninitializedParameter()
+        self.delta: torch.Tensor = UninitializedParameter()
+
+    def _initialize_parameters(self, x: torch.Tensor):
+        if x.ndim < 2:
+            raise ValueError(
+                f"Input tensor must have at least 2 dimensions (N, C), but got shape {tuple(x.shape)}"
+            )
+        param_shape = [1] * x.ndim
+        param_shape[1] = int(x.shape[1])
+        zeros = torch.zeros(param_shape, dtype=x.dtype, device=x.device)
+        with torch.no_grad():
+            for name in ("alpha", "beta", "gamma", "delta"):
+                setattr(self, name, Parameter(zeros.clone()))
+
+    def reset_parameters(self):
+        for name in ("alpha", "beta", "gamma", "delta"):
+            p = getattr(self, name)
+            if not isinstance(p, UninitializedParameter):
+                with torch.no_grad():
+                    p.zero_()
+
+    def forward(self, x: torch.Tensor):
+        if isinstance(self.alpha, UninitializedParameter):
+            self._initialize_parameters(x)
+
+        if x.ndim < 2:
+            raise ValueError(
+                f"Input tensor must have at least 2 dimensions (N, C), but got shape {tuple(x.shape)}"
+            )
+        if not isinstance(self.alpha, UninitializedParameter) and x.shape[1] != self.alpha.shape[1]:
+            raise RuntimeError(
+                f"GALU was initialized with C={self.alpha.shape[1]} but got C={x.shape[1]}. "
+                "Create a new GALU for a different channel size."
+            )
+
+        x_norm = x / self.max_input
+        zero = x.new_zeros(1)
+        part_prelu = F.relu(x_norm) + self.alpha * torch.minimum(x_norm, zero)
+        part_beta = self.beta * (
+            F.relu(1.0 - torch.abs(x_norm - 1.0))
+            + torch.minimum(torch.abs(x_norm - 3.0) - 1.0, zero)
+        )
+        part_gamma = self.gamma * (
+            F.relu(0.5 - torch.abs(x_norm - 0.5))
+            + torch.minimum(torch.abs(x_norm - 1.5) - 0.5, zero)
+        )
+        part_delta = self.delta * (
+            F.relu(0.5 - torch.abs(x_norm - 2.5))
+            + torch.minimum(torch.abs(x_norm - 3.5) - 0.5, zero)
+        )
+        z = part_prelu + part_beta + part_gamma + part_delta
+        return z * self.max_input

homa-0.2.0/src/homa/activations/classes/MELU.py

@@ -0,0 +1,70 @@
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+
+class MELU(nn.Module):
+    def __init__(self, maxInput: float = 1.0):
+        super().__init__()
+        self.maxInput = float(maxInput)
+        self._num_channels = None
+        self.register_parameter("alpha", None)
+        self.register_parameter("beta", None)
+        self.register_parameter("gamma", None)
+        self.register_parameter("delta", None)
+        self.register_parameter("xi", None)
+        self.register_parameter("psi", None)
+
+    def _ensure_parameters(self, x: torch.Tensor):
+        if x.dim() != 4:
+            raise ValueError(
+                f"Expected 4D input (N, C, H, W), got {x.dim()}D with shape {tuple(x.shape)}"
+            )
+        c = int(x.shape[1])
+        if self._num_channels is None:
+            self._num_channels = c
+        elif c != self._num_channels:
+            raise RuntimeError(
+                f"MELU was initialized with C={self._num_channels} but got C={c}. "
+                "Create a new MELU for a different channel size."
+            )
+
+        if self.alpha is None:
+            shape = (1, c, 1, 1)
+            device, dtype = x.device, x.dtype
+            for name in ("alpha", "beta", "gamma", "delta", "xi", "psi"):
+                setattr(
+                    self,
+                    name,
+                    nn.Parameter(torch.zeros(shape, dtype=dtype, device=device)),
+                )
+
+    def reset_parameters(self):
+        for p in (self.alpha, self.beta, self.gamma, self.delta, self.xi, self.psi):
+            if p is not None:
+                with torch.no_grad():
+                    p.zero_()
+
+    def forward(self, X: torch.Tensor) -> torch.Tensor:
+        self._ensure_parameters(X)
+
+        X_norm = X / self.maxInput
+        Y = torch.roll(X_norm, shifts=-1, dims=1)
+
+        term1 = F.relu(X_norm)
+        term2 = self.alpha * torch.clamp(X_norm, max=0)
+
+        dist_sq_beta = (X_norm - 2) ** 2 + (Y - 2) ** 2
+        dist_sq_gamma = (X_norm - 1) ** 2 + (Y - 1) ** 2
+        dist_sq_delta = (X_norm - 1) ** 2 + (Y - 3) ** 2
+        dist_sq_xi = (X_norm - 3) ** 2 + (Y - 1) ** 2
+        dist_sq_psi = (X_norm - 3) ** 2 + (Y - 3) ** 2
+
+        term3 = self.beta * torch.sqrt(F.relu(2 - dist_sq_beta))
+        term4 = self.gamma * torch.sqrt(F.relu(1 - dist_sq_gamma))
+        term5 = self.delta * torch.sqrt(F.relu(1 - dist_sq_delta))
+        term6 = self.xi * torch.sqrt(F.relu(1 - dist_sq_xi))
+        term7 = self.psi * torch.sqrt(F.relu(1 - dist_sq_psi))
+
+        Z_norm = term1 + term2 + term3 + term4 + term5 + term6 + term7
+        return Z_norm * self.maxInput

homa-0.2.0/src/homa/activations/classes/PDELU.py

@@ -0,0 +1,54 @@
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+
+class PDELU(nn.Module):
+    def __init__(self, theta: float = 0.5):
+        super().__init__()
+        if theta == 1.0:
+            raise ValueError(
+                "theta cannot be 1.0, as it would cause a division by zero."
+            )
+        self.theta = float(theta)
+        self._power_val = 1.0 / (1.0 - self.theta)
+        self.register_parameter("alpha", None)
+        self._num_channels = None
+
+    def _ensure_parameters(self, x: torch.Tensor):
+        if x.ndim < 2:
+            raise ValueError(
+                f"Input tensor must have at least 2 dimensions (N, C), but got shape {tuple(x.shape)}"
+            )
+
+        c = int(x.shape[1])
+        if self._num_channels is None:
+            self._num_channels = c
+        elif c != self._num_channels:
+            raise RuntimeError(
+                f"PDELU was initialized with C={self._num_channels} but got C={c}. "
+                "Create a new PDELU for a different channel size."
+            )
+
+        if self.alpha is None:
+            param_shape = [1] * x.ndim
+            param_shape[1] = c
+            self.alpha = nn.Parameter(
+                torch.full(param_shape, 0.1, dtype=x.dtype, device=x.device)
+            )
+
+    def reset_parameters(self):
+        if self.alpha is not None:
+            with torch.no_grad():
+                self.alpha.fill_(0.1)
+
+    def forward(self, x: torch.Tensor):
+        self._ensure_parameters(x)
+
+        positive_part = F.relu(x)
+        inner_term = F.relu(1.0 + (1.0 - self.theta) * x)
+        powered_term = torch.pow(inner_term, self._power_val)
+        subtracted_term = powered_term - 1.0
+        zero = torch.zeros(1, dtype=x.dtype, device=x.device)
+        negative_part = self.alpha * torch.minimum(subtracted_term, zero)
+        return positive_part + negative_part

homa-0.2.0/src/homa/activations/classes/SReLU.py

@@ -0,0 +1,69 @@
+import torch
+from torch import nn
+
+
+class SReLU(nn.Module):
+    def __init__(
+        self,
+        alpha_init: float = 0.0,
+        beta_init: float = 0.0,
+        gamma_init: float = 1.0,
+        delta_init: float = 1.0,
+    ):
+        super().__init__()
+        self.alpha_init_val = float(alpha_init)
+        self.beta_init_val = float(beta_init)
+        self.gamma_init_val = float(gamma_init)
+        self.delta_init_val = float(delta_init)
+        self._num_channels = None
+        self.register_parameter("alpha", None)
+        self.register_parameter("beta", None)
+        self.register_parameter("gamma", None)
+        self.register_parameter("delta", None)
+
+    def _ensure_parameters(self, x: torch.Tensor):
+        if x.dim() != 4:
+            raise ValueError(
+                f"Expected 4D input (N, C, H, W), got {x.dim()}D with shape {tuple(x.shape)}"
+            )
+        c = int(x.shape[1])
+        if self._num_channels is None:
+            self._num_channels = c
+        elif c != self._num_channels:
+            raise RuntimeError(
+                f"SReLU was initialized with C={self._num_channels} but got C={c}. "
+                "Create a new SReLU for different channel sizes."
+            )
+
+        if self.alpha is None:
+            shape = (1, c, 1, 1)
+            device, dtype = x.device, x.dtype
+            self.alpha = nn.Parameter(
+                torch.full(shape, self.alpha_init_val, dtype=dtype, device=device)
+            )
+            self.beta = nn.Parameter(
+                torch.full(shape, self.beta_init_val, dtype=dtype, device=device)
+            )
+            self.gamma = nn.Parameter(
+                torch.full(shape, self.gamma_init_val, dtype=dtype, device=device)
+            )
+            self.delta = nn.Parameter(
+                torch.full(shape, self.delta_init_val, dtype=dtype, device=device)
+            )
+
+    def reset_parameters(self):
+        if self.alpha is not None:
+            with torch.no_grad():
+                self.alpha.fill_(self.alpha_init_val)
+                self.beta.fill_(self.beta_init_val)
+                self.gamma.fill_(self.gamma_init_val)
+                self.delta.fill_(self.delta_init_val)
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self._ensure_parameters(x)
+
+        start = self.beta + self.alpha * (x - self.beta)
+        finish = self.delta + self.gamma * (x - self.delta)
+        out = torch.where(x < self.beta, start, x)
+        out = torch.where(x > self.delta, finish, out)
+        return out

homa-0.2.0/src/homa/activations/classes/SmallGALU.py

@@ -0,0 +1,58 @@
+import torch
+from torch import nn
+from torch.nn.parameter import Parameter
+import torch.nn.functional as F
+
+
+class SmallGALU(nn.Module):
+    def __init__(self, max_input: float = 1.0):
+        super().__init__()
+        if max_input <= 0:
+            raise ValueError("max_input must be positive.")
+        self.max_input = float(max_input)
+        self.register_parameter("alpha", None)
+        self.register_parameter("beta", None)
+        self._num_channels = None
+
+    def _initialize_parameters(self, x: torch.Tensor):
+        if x.ndim < 2:
+            raise ValueError(
+                f"Input tensor must have at least 2 dimensions (N, C), but got shape {tuple(x.shape)}"
+            )
+        self._num_channels = int(x.shape[1])
+        param_shape = [1] * x.ndim
+        param_shape[1] = self._num_channels
+        device = x.device
+        dtype = x.dtype
+        self.alpha = Parameter(torch.zeros(param_shape, dtype=dtype, device=device))
+        self.beta = Parameter(torch.zeros(param_shape, dtype=dtype, device=device))
+
+    def reset_parameters(self):
+        if self.alpha is not None:
+            with torch.no_grad():
+                self.alpha.zero_()
+                self.beta.zero_()
+
+    def forward(self, x: torch.Tensor):
+        if self.alpha is None:
+            self._initialize_parameters(x)
+        else:
+            if x.ndim < 2:
+                raise ValueError(
+                    f"Input tensor must have at least 2 dimensions (N, C), but got shape {tuple(x.shape)}"
+                )
+            if x.shape[1] != self._num_channels:
+                raise RuntimeError(
+                    f"SmallGALU was initialized with C={self._num_channels} but got C={x.shape[1]}. "
+                    "Create a new SmallGALU for a different channel size."
+                )
+
+        x_norm = x / self.max_input
+        zero = torch.zeros(1, dtype=x.dtype, device=x.device)
+        part_prelu = F.relu(x_norm) + self.alpha * torch.minimum(x_norm, zero)
+        part_beta = self.beta * (
+            F.relu(1.0 - torch.abs(x_norm - 1.0))
+            + torch.minimum(torch.abs(x_norm - 3.0) - 1.0, zero)
+        )
+        z = part_prelu + part_beta
+        return z * self.max_input

homa-0.2.0/src/homa/activations/classes/WideMELU.py

@@ -0,0 +1,90 @@
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+
+class WideMELU(nn.Module):
+    def __init__(self, maxInput: float = 1.0):
+        super().__init__()
+        self.maxInput = float(maxInput)
+        self._num_channels = None
+        self.register_parameter("alpha", None)
+        self.register_parameter("beta", None)
+        self.register_parameter("gamma", None)
+        self.register_parameter("delta", None)
+        self.register_parameter("xi", None)
+        self.register_parameter("psi", None)
+        self.register_parameter("theta", None)
+        self.register_parameter("lam", None)
+
+    def _ensure_parameters(self, x: torch.Tensor):
+        if x.dim() != 4:
+            raise ValueError(
+                f"Expected 4D input (N, C, H, W), got {x.dim()}D with shape {tuple(x.shape)}"
+            )
+
+        c = int(x.shape[1])
+        if self._num_channels is None:
+            self._num_channels = c
+        elif c != self._num_channels:
+            raise RuntimeError(
+                f"WideMELU was initialized with C={self._num_channels} but got C={c}. "
+                "Create a new WideMELU for different channel sizes."
+            )
+
+        if self.alpha is None:
+            shape = (1, c, 1, 1)
+            device, dtype = x.device, x.dtype
+            for name in (
+                "alpha",
+                "beta",
+                "gamma",
+                "delta",
+                "xi",
+                "psi",
+                "theta",
+                "lam",
+            ):
+                param = nn.Parameter(torch.zeros(shape, dtype=dtype, device=device))
+                setattr(self, name, param)
+
+    def reset_parameters(self):
+        params = (
+            self.alpha,
+            self.beta,
+            self.gamma,
+            self.delta,
+            self.xi,
+            self.psi,
+            self.theta,
+            self.lam,
+        )
+        for p in params:
+            if p is not None:
+                with torch.no_grad():
+                    p.zero_()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self._ensure_parameters(x)
+
+        X_norm = x / self.maxInput
+        Y = torch.roll(X_norm, shifts=-1, dims=1)
+
+        term1 = F.relu(X_norm)
+        term2 = self.alpha * torch.clamp(X_norm, max=0)
+        dist_sq_beta = (X_norm - 2) ** 2 + (Y - 2) ** 2
+        dist_sq_gamma = (X_norm - 1) ** 2 + (Y - 1) ** 2
+        dist_sq_delta = (X_norm - 1) ** 2 + (Y - 3) ** 2
+        dist_sq_xi = (X_norm - 3) ** 2 + (Y - 1) ** 2
+        dist_sq_psi = (X_norm - 3) ** 2 + (Y - 3) ** 2
+        dist_sq_theta = (X_norm - 1) ** 2 + (Y - 2) ** 2
+        dist_sq_lambda = (X_norm - 3) ** 2 + (Y - 2) ** 2
+        term3 = self.beta * torch.sqrt(F.relu(2 - dist_sq_beta))
+        term4 = self.gamma * torch.sqrt(F.relu(1 - dist_sq_gamma))
+        term5 = self.delta * torch.sqrt(F.relu(1 - dist_sq_delta))
+        term6 = self.xi * torch.sqrt(F.relu(1 - dist_sq_xi))
+        term7 = self.psi * torch.sqrt(F.relu(1 - dist_sq_psi))
+        term8 = self.theta * torch.sqrt(F.relu(1 - dist_sq_theta))
+        term9 = self.lam * torch.sqrt(F.relu(1 - dist_sq_lambda))
+        Z_norm = term1 + term2 + term3 + term4 + term5 + term6 + term7 + term8 + term9
+        return Z_norm * self.maxInput

{homa-0.1.94 → homa-0.2.0}/src/homa/activations/classes/__init__.py

@@ -5,4 +5,3 @@ from .MELU import MELU
 from .WideMELU import WideMELU
 from .PDELU import PDELU
 from .SReLU import SReLU
-from .StochasticActivation import StochasticActivation

homa-0.2.0/src/homa/vision/StochasticClassifier.py

@@ -0,0 +1,28 @@
+import torch
+import random
+from ..activations import APLU, GALU, SmallGALU, MELU, WideMELU, PDELU, SReLU
+
+
+class StochasticClassifier:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.pool = [
+            APLU,
+            GALU,
+            SmallGALU,
+            MELU,
+            WideMELU,
+            PDELU,
+            SReLU,
+            torch.nn.ReLU,
+            torch.nn.PReLU,
+            torch.nn.LeakyReLU,
+            torch.nn.ELU,
+        ]
+
+    def replace_activations(self, needle: torch.Tensor) -> None:
+        replacement = random.choice(self.pool)
+        for parent in self.network.modules():
+            for name, child in list(parent.named_children()):
+                if isinstance(child, needle):
+                    setattr(parent, name, replacement())

homa-0.2.0/src/homa/vision/StochasticResnet.py

@@ -0,0 +1,9 @@
+import torch
+from .Resnet import Resnet
+from .StochasticClassifier import StochasticClassifier
+
+
+class StochasticResnet(Resnet, StochasticClassifier):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.replace_activations(torch.nn.ReLU)

homa-0.2.0/src/homa/vision/StochasticSwin.py

@@ -0,0 +1,9 @@
+import torch
+from .Swin import Swin
+from .StochasticClassifier import StochasticClassifier
+
+
+class StochasticSwin(Swin, StochasticClassifier):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.replace_activations(torch.nn.GELU)

{homa-0.1.94 → homa-0.2.0}/src/homa.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: homa
-Version: 0.1.94
+Version: 0.2.0
 Summary: A curated list of machine learning and deep learning helpers.
 Author-email: Taha Shieenavaz <tahashieenavaz@gmail.com>
 Requires-Python: >=3.7

{homa-0.1.94 → homa-0.2.0}/src/homa.egg-info/SOURCES.txt

@@ -18,7 +18,6 @@ src/homa/activations/classes/MELU.py
 src/homa/activations/classes/PDELU.py
 src/homa/activations/classes/SReLU.py
 src/homa/activations/classes/SmallGALU.py
-src/homa/activations/classes/StochasticActivation.py
 src/homa/activations/classes/WideMELU.py
 src/homa/activations/classes/__init__.py
 src/homa/cli/HomaCommand.py
@@ -45,11 +44,11 @@ src/homa/torch/helpers.py
 src/homa/vision/ClassificationModel.py
 src/homa/vision/Model.py
 src/homa/vision/Resnet.py
+src/homa/vision/StochasticClassifier.py
 src/homa/vision/StochasticResnet.py
 src/homa/vision/StochasticSwin.py
 src/homa/vision/Swin.py
 src/homa/vision/__init__.py
-src/homa/vision/utils.py
 src/homa/vision/concerns/HasLabels.py
 src/homa/vision/concerns/HasLogits.py
 src/homa/vision/concerns/HasProbabilities.py

homa-0.1.94/src/homa/activations/classes/APLU.py

@@ -1,54 +0,0 @@
-import torch
-from ...device import get_device
-
-
-class APLU(torch.nn.Module):
-    def __init__(self, max_input: float = 1.0):
-        super(APLU, self).__init__()
-        self.max_input = max_input
-        self.alpha = None
-        self.beta = None
-        self.gamma = None
-        self.xi = None
-        self.psi = None
-        self.mu = None
-        self._num_channels = None
-        self.device = get_device()
-
-    def _initialize_parameters(self, x):
-        if x.ndim < 2:
-            raise ValueError(
-                f"Input tensor must have at least 2 dimensions (N, C), but got shape {x.shape}"
-            )
-
-        num_channels = x.shape[1]
-        self._num_channels = num_channels
-
-        param_shape = [1] * x.ndim
-        param_shape[1] = num_channels
-
-        self.alpha = torch.nn.Parameter(torch.zeros(param_shape), device=device())
-        self.beta = torch.nn.Parameter(torch.zeros(param_shape), device=device())
-        self.gamma = torch.nn.Parameter(torch.zeros(param_shape), device=device())
-
-        self.xi = torch.nn.Parameter(self.max_input * torch.rand(param_shape)).to(
-            self.device
-        )
-        self.psi = torch.nn.Parameter(self.max_input * torch.rand(param_shape)).to(
-            self.device
-        )
-        self.mu = torch.nn.Parameter(self.max_input * torch.rand(param_shape)).to(
-            self.device
-        )
-
-    def forward(self, x):
-        if self.alpha is None:
-            self._initialize_parameters(x)
-        a = torch.relu(x)
-
-        # following are called hinges
-        b = self.alpha * torch.relu(-x + self.xi)
-        c = self.beta * torch.relu(-x + self.psi)
-        d = self.gamma * torch.relu(-x + self.mu)
-        z = a + b + c + d
-        return z

homa-0.1.94/src/homa/activations/classes/GALU.py

@@ -1,53 +0,0 @@
-import torch
-from ...device import get_device
-
-
-class GALU(torch.nn.Module):
-    def __init__(self, max_input: float = 1.0):
-        super(GALU, self).__init__()
-        if max_input <= 0:
-            raise ValueError("max_input must be positive.")
-        self.max_input = max_input
-        self.alpha = None
-        self.beta = None
-        self.gamma = None
-        self.delta = None
-        self._num_channels = None
-        self.device = get_device()
-
-    def _initialize_parameters(self, x):
-        if x.ndim < 2:
-            raise ValueError(
-                f"Input tensor must have at least 2 dimensions (N, C), but got shape {x.shape}"
-            )
-
-        num_channels = x.shape[1]
-        self._num_channels = num_channels
-        param_shape = [1] * x.ndim
-        param_shape[1] = num_channels
-        self.alpha = torch.nn.Parameter(torch.zeros(param_shape)).to(self.device)
-        self.beta = torch.nn.Parameter(torch.zeros(param_shape)).to(self.device)
-        self.gamma = torch.nn.Parameter(torch.zeros(param_shape)).to(self.device)
-        self.delta = torch.nn.Parameter(torch.zeros(param_shape)).to(self.device)
-
-    def forward(self, x):
-        if self.alpha is None:
-            self._initialize_parameters(x)
-
-        zero = torch.tensor(0.0, device=x.device, dtype=x.dtype)
-        x_norm = x / self.max_input
-        part_prelu = torch.relu(x_norm) + self.alpha * torch.min(x_norm, zero)
-        part_beta = self.beta * (
-            torch.relu(1.0 - torch.abs(x_norm - 1.0))
-            + torch.min(torch.abs(x_norm - 3.0) - 1.0, zero)
-        )
-        part_gamma = self.gamma * (
-            torch.relu(0.5 - torch.abs(x_norm - 0.5))
-            + torch.min(torch.abs(x_norm - 1.5) - 0.5, zero)
-        )
-        part_delta = self.delta * (
-            torch.relu(0.5 - torch.abs(x_norm - 2.5))
-            + torch.min(torch.abs(x_norm - 3.5) - 0.5, zero)
-        )
-        z = part_prelu + part_beta + part_gamma + part_delta
-        return z * self.max_input

homa-0.1.94/src/homa/activations/classes/MELU.py

@@ -1,52 +0,0 @@
-import torch
-from ...device import get_device
-
-
-class MELU(torch.nn.Module):
-    def __init__(self, maxInput: float = 1.0):
-        super().__init__()
-        self.maxInput = float(maxInput)
-        self.alpha = None
-        self.beta = None
-        self.gamma = None
-        self.delta = None
-        self.xi = None
-        self.psi = None
-        self._initialized = False
-        self.device = get_device()
-
-    def _initialize_parameters(self, X: torch.Tensor):
-        if X.dim() != 4:
-            raise ValueError(
-                f"Expected 4D input (B, C, H, W), but got {X.dim()}D input."
-            )
-        num_channels = X.shape[1]
-        shape = (1, num_channels, 1, 1)
-        self.alpha = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.beta = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.gamma = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.delta = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.xi = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.psi = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self._initialized = True
-
-    def forward(self, X: torch.Tensor) -> torch.Tensor:
-        if not self._initialized:
-            self._initialize_parameters(X)
-        X_norm = X / self.maxInput
-        Y = torch.roll(X_norm, shifts=-1, dims=1)
-        term1 = torch.relu(X_norm)
-        term2 = self.alpha * torch.clamp(X_norm, max=0)
-        dist_sq_beta = (X_norm - 2) ** 2 + (Y - 2) ** 2
-        dist_sq_gamma = (X_norm - 1) ** 2 + (Y - 1) ** 2
-        dist_sq_delta = (X_norm - 1) ** 2 + (Y - 3) ** 2
-        dist_sq_xi = (X_norm - 3) ** 2 + (Y - 1) ** 2
-        dist_sq_psi = (X_norm - 3) ** 2 + (Y - 3) ** 2
-        term3 = self.beta * torch.sqrt(torch.relu(2 - dist_sq_beta))
-        term4 = self.gamma * torch.sqrt(torch.relu(1 - dist_sq_gamma))
-        term5 = self.delta * torch.sqrt(torch.relu(1 - dist_sq_delta))
-        term6 = self.xi * torch.sqrt(torch.relu(1 - dist_sq_xi))
-        term7 = self.psi * torch.sqrt(torch.relu(1 - dist_sq_psi))
-        Z_norm = term1 + term2 + term3 + term4 + term5 + term6 + term7
-        Z = Z_norm * self.maxInput
-        return Z

homa-0.1.94/src/homa/activations/classes/PDELU.py

@@ -1,41 +0,0 @@
-import torch
-from ...device import get_device
-
-
-class PDELU(torch.nn.Module):
-    def __init__(self, theta: float = 0.5):
-        super(PDELU, self).__init__()
-        if theta == 1.0:
-            raise ValueError(
-                "theta cannot be 1.0, as it would cause a division by zero."
-            )
-        self.theta = theta
-        self._power_val = 1.0 / (1.0 - self.theta)
-        self.alpha = torch.nn.UninitializedParameter()
-        self._num_channels = None
-        self.device = get_device()
-
-    def _initialize_parameters(self, x: torch.Tensor):
-        if x.ndim < 2:
-            raise ValueError(
-                f"Input tensor must have at least 2 dimensions (N, C), but got shape {x.shape}"
-            )
-
-        num_channels = x.shape[1]
-        self._num_channels = num_channels
-        param_shape = [1] * x.ndim
-        param_shape[1] = num_channels
-        init_tensor = torch.zeros(param_shape, device=self.device) + 0.1
-        self.alpha = torch.nn.Parameter(init_tensor).to(self.device)
-
-    def forward(self, x: torch.Tensor):
-        if self.alpha is None:
-            self._initialize_parameters(x)
-
-        zero = torch.tensor(0.0, device=self.device, dtype=x.dtype)
-        positive_part = torch.relu(x)
-        inner_term = torch.relu(1.0 + (1.0 - self.theta) * x)
-        powered_term = torch.pow(inner_term, self._power_val)
-        subtracted_term = powered_term - 1.0
-        negative_part = self.alpha * torch.min(subtracted_term, zero)
-        return positive_part + negative_part

homa-0.1.94/src/homa/activations/classes/SReLU.py

@@ -1,53 +0,0 @@
-import torch
-from ...device import get_device
-
-
-class SReLU(torch.nn.Module):
-    def __init__(
-        self,
-        alpha_init: float = 0.0,
-        beta_init: float = 0.0,
-        gamma_init: float = 1.0,
-        delta_init: float = 1.0,
-    ):
-        super().__init__()
-        self.alpha_init_val = alpha_init
-        self.beta_init_val = beta_init
-        self.gamma_init_val = gamma_init
-        self.delta_init_val = delta_init
-        self.alpha = torch.nn.UninitializedParameter()
-        self.beta = torch.nn.UninitializedParameter()
-        self.gamma = torch.nn.UninitializedParameter()
-        self.delta = torch.nn.UninitializedParameter()
-        self.device = get_device()
-
-    def _initialize_parameters(self, x: torch.Tensor):
-        if isinstance(self.alpha, torch.nn.UninitializedParameter):
-            if x.dim() < 2:
-                raise ValueError(
-                    f"Input tensor must have at least 2 dimensions (N, C), but got {x.dim()}"
-                )
-
-            num_channels = x.shape[1]
-            param_shape = [1] * x.dim()
-            param_shape[1] = num_channels
-            self.alpha = torch.nn.Parameter(
-                torch.full(param_shape, self.alpha_init_val)
-            ).to(self.device)
-            self.beta = torch.nn.Parameter(
-                torch.full(param_shape, self.beta_init_val)
-            ).to(self.device)
-            self.gamma = torch.nn.Parameter(
-                torch.full(param_shape, self.gamma_init_val)
-            ).to(self.device)
-            self.delta = torch.nn.Parameter(
-                torch.full(param_shape, self.delta_init_val)
-            ).to(self.device)
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        self._initialize_parameters(x)
-        start = self.beta + self.alpha * (x - self.beta)
-        finish = self.delta + self.gamma * (x - self.delta)
-        out = torch.where(x < self.beta, start, x)
-        out = torch.where(x > self.delta, finish, out)
-        return out

homa-0.1.94/src/homa/activations/classes/SmallGALU.py

@@ -1,41 +0,0 @@
-import torch
-from ...device import get_device
-
-
-class SmallGALU(torch.nn.Module):
-    def __init__(self, max_input: float = 1.0):
-        super(SmallGALU, self).__init__()
-        if max_input <= 0:
-            raise ValueError("max_input must be positive.")
-        self.max_input = max_input
-        self.alpha = None
-        self.beta = None
-        self._num_channels = None
-        self.device = get_device()
-
-    def _initialize_parameters(self, x):
-        if x.ndim < 2:
-            raise ValueError(
-                f"Input tensor must have at least 2 dimensions (N, C), but got shape {x.shape}"
-            )
-
-        num_channels = x.shape[1]
-        self._num_channels = num_channels
-        param_shape = [1] * x.ndim
-        param_shape[1] = num_channels
-        self.alpha = torch.nn.Parameter(torch.zeros(param_shape)).to(self.device)
-        self.beta = torch.nn.Parameter(torch.zeros(param_shape)).to(self.device)
-
-    def forward(self, x):
-        if self.alpha is None:
-            self._initialize_parameters(x)
-
-        zero = torch.tensor(0.0, device=x.device, dtype=x.dtype)
-        x_norm = x / self.max_input
-        part_prelu = torch.relu(x_norm) + self.alpha * torch.min(x_norm, zero)
-        part_beta = self.beta * (
-            torch.relu(1.0 - torch.abs(x_norm - 1.0))
-            + torch.min(torch.abs(x_norm - 3.0) - 1.0, zero)
-        )
-        z = part_prelu + part_beta
-        return z * self.max_input

homa-0.1.94/src/homa/activations/classes/StochasticActivation.py

@@ -1,34 +0,0 @@
-import torch
-import random
-
-from .APLU import APLU
-from .GALU import GALU
-from .SmallGALU import SmallGALU
-from .MELU import MELU
-from .WideMELU import WideMELU
-from .PDELU import PDELU
-from .SReLU import SReLU
-
-
-class StochasticActivation(torch.nn.Module):
-    def __init__(self):
-        super().__init__()
-        self.gate = random.choice(
-            [
-                APLU,
-                GALU,
-                SmallGALU,
-                MELU,
-                WideMELU,
-                PDELU,
-                SReLU,
-                torch.nn.ReLU,
-                torch.nn.PReLU,
-                torch.nn.LeakyReLU,
-                torch.nn.ELU,
-            ]
-        )
-        self.gate = self.gate()
-
-    def forward(self, x: torch.Tensor) -> torch.Tensor:
-        return self.gate(x)

homa-0.1.94/src/homa/activations/classes/WideMELU.py

@@ -1,63 +0,0 @@
-import torch
-from ...device import get_device
-
-
-class WideMELU(torch.nn.Module):
-    def __init__(self, maxInput: float = 1.0):
-        super().__init__()
-        self.maxInput = float(maxInput)
-        self.alpha = None
-        self.beta = None
-        self.gamma = None
-        self.delta = None
-        self.xi = None
-        self.psi = None
-        self.theta = None
-        self.lam = None
-        self._initialized = False
-        self.device = get_device()
-
-    def _initialize_parameters(self, X: torch.Tensor):
-        if X.dim() != 4:
-            raise ValueError(
-                f"Expected 4D input (B, C, H, W), but got {X.dim()}D input."
-            )
-
-        num_channels = X.shape[1]
-        shape = (1, num_channels, 1, 1)
-
-        self.alpha = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.beta = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.gamma = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.delta = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.xi = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.psi = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.theta = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self.lam = torch.nn.Parameter(torch.zeros(shape)).to(self.device)
-        self._initialized = True
-
-    def forward(self, X: torch.Tensor) -> torch.Tensor:
-        if not self._initialized:
-            self._initialize_parameters(X)
-        X_norm = X / self.maxInput
-        Y = torch.roll(X_norm, shifts=-1, dims=1)
-        term1 = torch.relu(X_norm)
-        term2 = self.alpha * torch.clamp(X_norm, max=0)
-        dist_sq_beta = (X_norm - 2) ** 2 + (Y - 2) ** 2
-        dist_sq_gamma = (X_norm - 1) ** 2 + (Y - 1) ** 2
-        dist_sq_delta = (X_norm - 1) ** 2 + (Y - 3) ** 2
-        dist_sq_xi = (X_norm - 3) ** 2 + (Y - 1) ** 2
-        dist_sq_psi = (X_norm - 3) ** 2 + (Y - 3) ** 2
-        dist_sq_theta = (X_norm - 1) ** 2 + (Y - 2) ** 2
-        dist_sq_lambda = (X_norm - 3) ** 2 + (Y - 2) ** 2
-
-        term3 = self.beta * torch.sqrt(torch.relu(2 - dist_sq_beta))
-        term4 = self.gamma * torch.sqrt(torch.relu(1 - dist_sq_gamma))
-        term5 = self.delta * torch.sqrt(torch.relu(1 - dist_sq_delta))
-        term6 = self.xi * torch.sqrt(torch.relu(1 - dist_sq_xi))
-        term7 = self.psi * torch.sqrt(torch.relu(1 - dist_sq_psi))
-        term8 = self.theta * torch.sqrt(torch.relu(1 - dist_sq_theta))
-        term9 = self.lam * torch.sqrt(torch.relu(1 - dist_sq_lambda))
-        Z_norm = term1 + term2 + term3 + term4 + term5 + term6 + term7 + term8 + term9
-        Z = Z_norm * self.maxInput
-        return Z

homa-0.1.94/src/homa/vision/StochasticResnet.py

@@ -1,9 +0,0 @@
-from .Resnet import Resnet
-from .utils import replace_relu
-from ..activations import StochasticActivation
-
-
-class StochasticResnet(Resnet):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        replace_relu(self.network, StochasticActivation)

homa-0.1.94/src/homa/vision/StochasticSwin.py

@@ -1,9 +0,0 @@
-from .Swin import Swin
-from .utils import replace_gelu
-from ..activations import StochasticActivation
-
-
-class StochasticSwin(Swin):
-    def __init__(self, *args, **kwargs):
-        super().__init__(*args, **kwargs)
-        replace_gelu(self.network.encoder, StochasticActivation)

homa-0.1.94/src/homa/vision/utils.py

@@ -1,25 +0,0 @@
-import torch
-
-
-def replace_modules(
-    model: torch.nn.Module, find: list | torch.Tensor, replacement: torch.nn.Module
-) -> int:
-    if not isinstance(find, list):
-        find = [find]
-
-    replaced = 0
-    for parent in model.modules():
-        for name, child in list(parent.named_children()):
-            for needle in find:
-                if isinstance(child, needle):
-                    setattr(parent, name, replacement())
-                    replaced += 1
-    return replaced
-
-
-def replace_relu(model: torch.nn.Module, replacement: torch.nn.Module):
-    return replace_modules(model, torch.nn.ReLU, replacement)
-
-
-def replace_gelu(model: torch.nn.Module, replacement: torch.nn.Module):
-    return replace_modules(model, torch.nn.GELU, replacement)