homa 0.1.1__py3-none-any.whl → 0.2.0__py3-none-any.whl

homa/activations/classes/WideMELU.py

@@ -1,45 +1,76 @@
 import torch
+from torch import nn
+import torch.nn.functional as F
 
 
-class WideMELU(torch.nn.Module):
+class WideMELU(nn.Module):
     def __init__(self, maxInput: float = 1.0):
         super().__init__()
         self.maxInput = float(maxInput)
-        self.alpha = None
-        self.beta = None
-        self.gamma = None
-        self.delta = None
-        self.xi = None
-        self.psi = None
-        self.theta = None
-        self.lam = None
-        self._initialized = False
-
-    def _initialize_parameters(self, X: torch.Tensor):
-        if X.dim() != 4:
+        self._num_channels = None
+        self.register_parameter("alpha", None)
+        self.register_parameter("beta", None)
+        self.register_parameter("gamma", None)
+        self.register_parameter("delta", None)
+        self.register_parameter("xi", None)
+        self.register_parameter("psi", None)
+        self.register_parameter("theta", None)
+        self.register_parameter("lam", None)
+
+    def _ensure_parameters(self, x: torch.Tensor):
+        if x.dim() != 4:
             raise ValueError(
-                f"Expected 4D input (B, C, H, W), but got {X.dim()}D input."
+                f"Expected 4D input (N, C, H, W), got {x.dim()}D with shape {tuple(x.shape)}"
+            )
+
+        c = int(x.shape[1])
+        if self._num_channels is None:
+            self._num_channels = c
+        elif c != self._num_channels:
+            raise RuntimeError(
+                f"WideMELU was initialized with C={self._num_channels} but got C={c}. "
+                "Create a new WideMELU for different channel sizes."
             )
 
-        num_channels = X.shape[1]
-        shape = (1, num_channels, 1, 1)
-
-        self.alpha = torch.nn.Parameter(torch.zeros(shape))
-        self.beta = torch.nn.Parameter(torch.zeros(shape))
-        self.gamma = torch.nn.Parameter(torch.zeros(shape))
-        self.delta = torch.nn.Parameter(torch.zeros(shape))
-        self.xi = torch.nn.Parameter(torch.zeros(shape))
-        self.psi = torch.nn.Parameter(torch.zeros(shape))
-        self.theta = torch.nn.Parameter(torch.zeros(shape))
-        self.lam = torch.nn.Parameter(torch.zeros(shape))
-        self._initialized = True
-
-    def forward(self, X: torch.Tensor) -> torch.Tensor:
-        if not self._initialized:
-            self._initialize_parameters(X)
-        X_norm = X / self.maxInput
+        if self.alpha is None:
+            shape = (1, c, 1, 1)
+            device, dtype = x.device, x.dtype
+            for name in (
+                "alpha",
+                "beta",
+                "gamma",
+                "delta",
+                "xi",
+                "psi",
+                "theta",
+                "lam",
+            ):
+                param = nn.Parameter(torch.zeros(shape, dtype=dtype, device=device))
+                setattr(self, name, param)
+
+    def reset_parameters(self):
+        params = (
+            self.alpha,
+            self.beta,
+            self.gamma,
+            self.delta,
+            self.xi,
+            self.psi,
+            self.theta,
+            self.lam,
+        )
+        for p in params:
+            if p is not None:
+                with torch.no_grad():
+                    p.zero_()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self._ensure_parameters(x)
+
+        X_norm = x / self.maxInput
         Y = torch.roll(X_norm, shifts=-1, dims=1)
-        term1 = torch.relu(X_norm)
+
+        term1 = F.relu(X_norm)
         term2 = self.alpha * torch.clamp(X_norm, max=0)
         dist_sq_beta = (X_norm - 2) ** 2 + (Y - 2) ** 2
         dist_sq_gamma = (X_norm - 1) ** 2 + (Y - 1) ** 2
@@ -48,14 +79,12 @@ class WideMELU(torch.nn.Module):
         dist_sq_psi = (X_norm - 3) ** 2 + (Y - 3) ** 2
         dist_sq_theta = (X_norm - 1) ** 2 + (Y - 2) ** 2
         dist_sq_lambda = (X_norm - 3) ** 2 + (Y - 2) ** 2
-
-        term3 = self.beta * torch.sqrt(torch.relu(2 - dist_sq_beta))
-        term4 = self.gamma * torch.sqrt(torch.relu(1 - dist_sq_gamma))
-        term5 = self.delta * torch.sqrt(torch.relu(1 - dist_sq_delta))
-        term6 = self.xi * torch.sqrt(torch.relu(1 - dist_sq_xi))
-        term7 = self.psi * torch.sqrt(torch.relu(1 - dist_sq_psi))
-        term8 = self.theta * torch.sqrt(torch.relu(1 - dist_sq_theta))
-        term9 = self.lam * torch.sqrt(torch.relu(1 - dist_sq_lambda))
+        term3 = self.beta * torch.sqrt(F.relu(2 - dist_sq_beta))
+        term4 = self.gamma * torch.sqrt(F.relu(1 - dist_sq_gamma))
+        term5 = self.delta * torch.sqrt(F.relu(1 - dist_sq_delta))
+        term6 = self.xi * torch.sqrt(F.relu(1 - dist_sq_xi))
+        term7 = self.psi * torch.sqrt(F.relu(1 - dist_sq_psi))
+        term8 = self.theta * torch.sqrt(F.relu(1 - dist_sq_theta))
+        term9 = self.lam * torch.sqrt(F.relu(1 - dist_sq_lambda))
         Z_norm = term1 + term2 + term3 + term4 + term5 + term6 + term7 + term8 + term9
-        Z = Z_norm * self.maxInput
-        return Z
+        return Z_norm * self.maxInput

homa/activations/classes/__init__.py

@@ -5,4 +5,3 @@ from .MELU import MELU
 from .WideMELU import WideMELU
 from .PDELU import PDELU
 from .SReLU import SReLU
-from .StochasticActivation import StochasticActivation

homa/ensemble/Ensemble.py

@@ -1,8 +1,7 @@
 from .concerns import (
     ReportsSize,
-    RecordsStateDictionaries,
+    StoresModels,
     ReportsClassificationMetrics,
-    HasNetwork,
     PredictsProbabilities,
 )
 
@@ -10,9 +9,8 @@ from .concerns import (
 class Ensemble(
     ReportsSize,
     ReportsClassificationMetrics,
-    RecordsStateDictionaries,
     PredictsProbabilities,
-    HasNetwork,
+    StoresModels,
 ):
     def __init__(self):
         super().__init__()

homa/ensemble/concerns/CalculatesMetricNecessities.py

@@ -1,20 +1,24 @@
 import torch
+from ...device import get_device
 
 
 class CalculatesMetricNecessities:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+    @torch.no_grad()
     def metric_necessities(self, dataloader):
-        all_predictions = []
-        all_labels = []
+        predictions, labels = [], []
+        device = get_device()
         for x, y in dataloader:
-            batch_logits_list = []
+            x, y = x.to(device), y.to(device)
+            sum_logits = None
             for model in self.models:
-                batch_logits_list.append(model(x))
-            all_batch_logits = torch.stack(batch_logits_list)
-            avg_logits = torch.mean(all_batch_logits, dim=0)
-            _, preds = torch.max(avg_logits, 1)
-            all_predictions.extend(preds.cpu().numpy())
-            all_labels.extend(y.cpu().numpy())
-        return all_predictions, all_labels
+                model.to(device)
+                model.eval()
+                logits = model(x)
+                sum_logits = logits if sum_logits is None else sum_logits + logits
+            batch_predictions = sum_logits.argmax(dim=1)
+            predictions.extend(batch_predictions.cpu().numpy())
+            labels.extend(y.cpu().numpy())
+        return predictions, labels

homa/ensemble/concerns/PredictsProbabilities.py

@@ -9,3 +9,7 @@ class PredictsProbabilities(ReportsLogits):
     def predict(self, x: torch.Tensor) -> torch.Tensor:
         logits = self.logits(x)
         return torch.nn.functional.softmax(logits, dim=1)
+
+    @torch.no_grad()
+    def predict_(self, x: torch.Tensor) -> torch.Tensor:
+        return self.predict(x)

homa/ensemble/concerns/ReportsClassificationMetrics.py

@@ -6,8 +6,8 @@ from .CalculatesMetricNecessities import CalculatesMetricNecessities
 
 class ReportsClassificationMetrics(
     CalculatesMetricNecessities,
-    ReportsEnsembleF1,
     ReportsEnsembleAccuracy,
+    ReportsEnsembleF1,
     ReportsEnsembleKappa,
 ):
     pass

homa/ensemble/concerns/ReportsEnsembleAccuracy.py

@@ -1,10 +1,11 @@
 from sklearn.metrics import accuracy_score as accuracy
+from torch.utils.data import DataLoader
 
 
 class ReportsEnsembleAccuracy:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def accuracy(self) -> float:
-        predictions, labels = self.metric_necessities()
+    def accuracy(self, dataloader: DataLoader) -> float:
+        predictions, labels = self.metric_necessities(dataloader)
         return accuracy(labels, predictions)

homa/ensemble/concerns/ReportsLogits.py

@@ -11,3 +11,7 @@ class ReportsLogits:
         for model in self.models:
             logits += model(x)
         return logits
+
+    @torch.no_grad()
+    def logits_(self, *args, **kwargs):
+        return self.logits(*args, **kwargs)

homa/ensemble/concerns/ReportsSize.py

@@ -4,8 +4,8 @@ class ReportsSize:
 
     @property
     def size(self):
-        return len(self.state_dicts)
+        return len(self.models)
 
     @property
     def length(self):
-        return len(self.state_dicts)
+        return len(self.models)

homa/ensemble/concerns/StoresModels.py

@@ -0,0 +1,29 @@
+import torch
+from copy import deepcopy
+from typing import List
+from ...vision import Model
+
+
+class StoresModels:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.models: List[torch.nn.Module] = []
+
+    def record(self, model: Model | torch.nn.Module):
+        model_: torch.nn.Module | None = None
+        if isinstance(model, Model):
+            model_ = deepcopy(model.network)
+        elif isinstance(model, torch.nn.Module):
+            model_ = deepcopy(model)
+        else:
+            raise TypeError("Wrong input to ensemble record")
+        self.models.append(model_)
+
+    def push(self, *args, **kwargs):
+        self.record(*args, **kwargs)
+
+    def append(self, *args, **kwargs):
+        self.record(*args, **kwargs)
+
+    def add(self, *args, **kwargs):
+        self.record(*args, **kwargs)

homa/ensemble/concerns/__init__.py

@@ -1,10 +1,9 @@
 from .CalculatesMetricNecessities import CalculatesMetricNecessities
-from .HasNetwork import HasNetwork
 from .PredictsProbabilities import PredictsProbabilities
-from .RecordsStateDictionaries import RecordsStateDictionaries
 from .ReportsClassificationMetrics import ReportsClassificationMetrics
 from .ReportsEnsembleAccuracy import ReportsEnsembleAccuracy
 from .ReportsEnsembleF1 import ReportsEnsembleF1
 from .ReportsEnsembleKappa import ReportsEnsembleKappa
 from .ReportsLogits import ReportsLogits
 from .ReportsSize import ReportsSize
+from .StoresModels import StoresModels

homa/loss/LogitNormLoss.py

@@ -0,0 +1,12 @@
+import torch
+from .Loss import Loss
+
+
+class LogitNormLoss(Loss):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def forward(self, logits, target):
+        norms = torch.norm(logits, p=2, dim=-1, keepdim=True) + 1e-7
+        normalized_logits = torch.div(logits, norms)
+        return torch.nn.functional.cross_entropy(normalized_logits, target)

homa/loss/Loss.py

@@ -0,0 +1,2 @@
+class Loss:
+    pass

homa/loss/__init__.py

@@ -0,0 +1,2 @@
+from .LogitNormLoss import LogitNormLoss
+from .Loss import Loss

homa/torch/__init__.py

@@ -1,2 +1 @@
-from .Module import Module
 from .helpers import *

homa/vision/ClassificationModel.py

@@ -0,0 +1,5 @@
+from .Model import Model
+
+
+class ClassificationModel(Model):
+    pass

homa/vision/Resnet.py

@@ -1,12 +1,13 @@
 import torch
 from .modules import ResnetModule
-from .Model import Model
-from .concerns import Trainable
+from .ClassificationModel import ClassificationModel
+from .concerns import Trainable, ReportsMetrics
+from ..device import get_device
 
 
-class Resnet(Model, Trainable):
-    def __init__(self, num_classes: int, lr: float):
+class Resnet(ClassificationModel, Trainable, ReportsMetrics):
+    def __init__(self, num_classes: int, lr: float = 0.001):
         super().__init__()
-        self.network = ResnetModule(num_classes)
+        self.network = ResnetModule(num_classes).to(get_device())
         self.criterion = torch.nn.CrossEntropyLoss()
         self.optimizer = torch.optim.SGD(self.network.parameters(), lr=lr, momentum=0.9)

homa/vision/StochasticClassifier.py

@@ -0,0 +1,28 @@
+import torch
+import random
+from ..activations import APLU, GALU, SmallGALU, MELU, WideMELU, PDELU, SReLU
+
+
+class StochasticClassifier:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.pool = [
+            APLU,
+            GALU,
+            SmallGALU,
+            MELU,
+            WideMELU,
+            PDELU,
+            SReLU,
+            torch.nn.ReLU,
+            torch.nn.PReLU,
+            torch.nn.LeakyReLU,
+            torch.nn.ELU,
+        ]
+
+    def replace_activations(self, needle: torch.Tensor) -> None:
+        replacement = random.choice(self.pool)
+        for parent in self.network.modules():
+            for name, child in list(parent.named_children()):
+                if isinstance(child, needle):
+                    setattr(parent, name, replacement())

homa/vision/StochasticResnet.py

@@ -1,8 +1,9 @@
-from .modules import StochasticResnetModule
+import torch
 from .Resnet import Resnet
+from .StochasticClassifier import StochasticClassifier
 
 
-class StochasticResnet(Resnet):
-    def __init__(self, **kwargs):
-        super().__init__(**kwargs)
-        self.model = StochasticResnetModule(kwargs["num_classes"])
+class StochasticResnet(Resnet, StochasticClassifier):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.replace_activations(torch.nn.ReLU)

homa/vision/StochasticSwin.py

@@ -0,0 +1,9 @@
+import torch
+from .Swin import Swin
+from .StochasticClassifier import StochasticClassifier
+
+
+class StochasticSwin(Swin, StochasticClassifier):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.replace_activations(torch.nn.GELU)

homa/vision/Swin.py

@@ -0,0 +1,12 @@
+import torch
+from .ClassificationModel import ClassificationModel
+from .concerns import Trainable, ReportsMetrics
+from .modules import SwinModule
+
+
+class Swin(ClassificationModel, Trainable, ReportsMetrics):
+    def __init__(self, num_classes: int, lr: float = 0.0001):
+        super().__init__()
+        self.network = SwinModule(num_classes=num_classes)
+        self.optimizer = torch.optim.AdamW(self.network.parameters(), lr=lr)
+        self.criterion = torch.nn.CrossEntropyLoss()

homa/vision/__init__.py

@@ -1,3 +1,4 @@
 from .Model import Model
 from .Resnet import Resnet
 from .StochasticResnet import StochasticResnet
+from .Swin import Swin

homa/vision/concerns/HasLabels.py

@@ -0,0 +1,13 @@
+import torch
+
+
+class HasLabels:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def predict(self, x: torch.Tensor):
+        return torch.argmax(self.logits(x), dim=1)
+
+    @torch.no_grad()
+    def predict_(self, x: torch.Tensor):
+        return torch.argmax(self.logits(x), dim=1)

homa/vision/concerns/HasLogits.py

@@ -0,0 +1,12 @@
+import torch
+
+
+class HasLogits:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def logits(self, x: torch.Tensor) -> torch.Tensor:
+        return self.network(x)
+
+    def logits_(self, x: torch.Tensor) -> torch.Tensor:
+        return self.network(x)

homa/vision/concerns/HasProbabilities.py

@@ -0,0 +1,9 @@
+import torch
+
+
+class HasProbabilities:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def probabilities(self, x: torch.Tensor):
+        return torch.softmax(self.logits(x), dim=1)

homa/vision/concerns/ReportsAccuracy.py

@@ -0,0 +1,27 @@
+from torch import Tensor, no_grad
+from torch.utils.data.dataloader import DataLoader
+from ...device import get_device
+
+
+class ReportsAccuracy:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def accuracy_tensors(self, x: Tensor, y: Tensor) -> float:
+        predictions = self.predict_(x)
+        return (predictions == y).float().mean().item()
+
+    def accuracy_dataloader(self, dataloader: DataLoader):
+        correct, total = 0, 0
+        for x, y in dataloader:
+            x, y = x.to(get_device()), y.to(get_device())
+            predictions = self.predict_(x)
+            correct += (predictions == y).sum().item()
+            total += y.numel()
+        return correct / total if total > 0 else 0.0
+
+    def accuracy(self, x: Tensor | DataLoader, y: Tensor | None = None) -> float:
+        self.network.eval()
+        if isinstance(x, DataLoader):
+            return self.accuracy_dataloader(x)
+        return self.accuracy_tensors(x, y)

homa/vision/concerns/ReportsMetrics.py

@@ -0,0 +1,6 @@
+from .ReportsAccuracy import ReportsAccuracy
+
+
+class ReportsMetrics(ReportsAccuracy):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)

homa/vision/concerns/Trainable.py

@@ -1,9 +1,12 @@
 from torch import Tensor
 from torch.utils.data.dataloader import DataLoader
+from .HasLogits import HasLogits
+from .HasProbabilities import HasProbabilities
+from .HasLabels import HasLabels
 from ...device import get_device
 
 
-class Trainable:
+class Trainable(HasLogits, HasProbabilities, HasLabels):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
@@ -16,7 +19,7 @@ class Trainable:
     def train_tensors(self, x: Tensor, y: Tensor):
         self.network.train()
         self.optimizer.zero_grad()
-        loss = self.criterion(y, self.network(x).float())
+        loss = self.criterion(self.network(x).float(), y)
         loss.backward()
         self.optimizer.step()
 

homa/vision/concerns/__init__.py

@@ -1 +1,6 @@
+from .HasLabels import HasLabels
+from .HasLogits import HasLogits
+from .HasProbabilities import HasProbabilities
+from .ReportsAccuracy import ReportsAccuracy
+from .ReportsMetrics import ReportsMetrics
 from .Trainable import Trainable

homa/vision/modules/SwinModule.py

@@ -0,0 +1,23 @@
+import torch
+from torchvision.models import swin_v2_b
+from torch.nn.init import kaiming_uniform_ as kaiming
+
+
+class SwinModule(torch.nn.Module):
+    def __init__(self, num_classes: int):
+        super().__init__()
+        self.num_classes = num_classes
+        self._create_encoder()
+        self._create_fc()
+
+    def _create_encoder(self):
+        self.encoder = swin_v2_b(weights="DEFAULT")
+        self.encoder.head = torch.nn.Identity()
+
+    def _create_fc(self):
+        self.fc = torch.nn.Linear(1024, self.num_classes)
+        kaiming(self.fc.weight, mode="fan_in", nonlinearity="relu")
+
+    def forward(self, images: torch.Tensor):
+        features = self.encoder(images)
+        return self.fc(features)

homa/vision/modules/__init__.py

@@ -1,2 +1,2 @@
 from .ResnetModule import ResnetModule
-from .StochasticResnetModule import StochasticResnetModule
+from .SwinModule import SwinModule

homa-0.2.0.dist-info/METADATA

@@ -0,0 +1,75 @@
+Metadata-Version: 2.4
+Name: homa
+Version: 0.2.0
+Summary: A curated list of machine learning and deep learning helpers.
+Author-email: Taha Shieenavaz <tahashieenavaz@gmail.com>
+Requires-Python: >=3.7
+Description-Content-Type: text/markdown
+Requires-Dist: numpy
+Requires-Dist: torch
+Requires-Dist: fire
+
+# Core
+
+### Device Management
+
+```py
+from homa import cpu, mps, cuda, device
+
+torch.tensor([1, 2, 3, 4, 5]).to(cpu())
+torch.tensor([1, 2, 3, 4, 5]).to(cuda())
+torch.tensor([1, 2, 3, 4, 5]).to(mps())
+torch.tensor([1, 2, 3, 4, 5]).to(device())
+```
+
+# Vision
+
+## Resnet
+
+This is the standard ResNet50 module.
+
+You can train the model with a `DataLoader` object.
+
+```py
+from homa.vision import Resnet
+
+model = Resnet(num_classes=10, lr=0.001)
+for epoch in range(10):
+    model.train(train_dataloader)
+```
+
+Similarly you can manually take care of decomposition of data from the `DataLoader`.
+
+```py
+from homa.vision import Resnet
+
+model = Resnet(num_classes=10, lr=0.001)
+for epoch in range(10):
+    for x, y in train_dataloader:
+        model.train(x, y)
+```
+
+## StochasticResnet
+
+This is a ResNet module whose activation functions are replaced from a pool of different activation functions randomly. Read more on the [(paper)](https://www.mdpi.com/1424-8220/22/16/6129).
+
+You can train the model with a `DataLoader` object.
+
+```py
+from homa.vision import StochasticResnet
+
+model = StochasticResnet(num_classes=10, lr=0.001)
+for epoch in range(10):
+    model.train(train_dataloader)
+```
+
+Similarly you can manually take care of decomposition of data from the `DataLoader`.
+
+```py
+from homa.vision import StochasticResnet
+
+model = StochasticResnet(num_classes=10, lr=0.001)
+for epoch in range(10):
+    for x, y in train_dataloader:
+        model.train(x, y)
+```