homa 0.1.1__py3-none-any.whl → 0.2.9__py3-none-any.whl

homa/activations/TBSReLU.py

@@ -0,0 +1,13 @@
+import torch
+from .ActivationFunction import ActivationFunction
+
+
+class TBSReLU(ActivationFunction):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        a = 1 - torch.exp(-x)
+        b = 1 + torch.exp(-x)
+        c = a / b
+        return x * torch.tanh(c)

homa/activations/TSReLU.py

@@ -0,0 +1,10 @@
+import torch
+from .ActivationFunction import ActivationFunction
+
+
+class TSReLU(ActivationFunction):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x * torch.tanh(torch.sigmoid(x))

homa/activations/TangentBipolarSigmoidReLU.py

@@ -0,0 +1,6 @@
+from .TBSReLU import TBSReLU
+
+
+class TangentBipolarSigmoidReLU(TBSReLU):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)

homa/activations/TangentSigmoidReLU.py

@@ -0,0 +1,6 @@
+from .TSReLU import TSReLU
+
+
+class TangentSigmoidReLU(TSReLU):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)

homa/activations/TeLU.py

@@ -0,0 +1,9 @@
+import torch
+
+
+class TeLU(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor):
+        return x * torch.tanh(torch.exp(x))

homa/activations/TripleStateSwish.py

@@ -0,0 +1,15 @@
+import torch
+from .ActivationFunction import ActivationFunction
+
+
+class TripleStateSwish(ActivationFunction):
+    def __init__(self, alpha: float = 20, beta: float = 40, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.alpha = alpha
+        self.beta = beta
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        a = 1 / (1 + torch.exp(-x))
+        b = 1 / (1 + torch.exp(-x + self.alpha))
+        c = 1 / (1 + torch.exp(-x + self.beta))
+        return x * a * (a + b + c)

homa/activations/WideMeLU.py

@@ -0,0 +1,15 @@
+from .MexicanReLU import MexicanReLU
+
+
+class WideMeLU(MexicanReLU):
+    def __init__(self, channels: int | None = None, max_input: float = 1.0):
+        self.hats = [
+            (2.0, 2.0),
+            (1.0, 1.0),
+            (3.0, 1.0),
+            (0.5, 0.5),
+            (1.5, 0.5),
+            (2.5, 0.5),
+            (3.5, 0.5),
+        ]
+        super().__init__(self.hats, channels=channels, max_input=max_input)

homa/activations/__init__.py

@@ -1,2 +1,49 @@
-from .classes import *
-from .utils import *
+from .ShiftedReLU import ShiftedReLU
+from .PLogish import PLogish
+from .ParametricLogish import ParametricLogish
+from .ExpExpish import ExpExpish
+from .GeneralizedSwish import GeneralizedSwish
+from .TBSReLU import TBSReLU
+from .NoisyReLU import NoisyReLU
+from .ExponentialDLReLU import ExponentialDLReLU
+from .SReLU import SReLU
+from .TangentSigmoidReLU import TangentSigmoidReLU
+from .Phish import Phish
+from .WideMeLU import WideMeLU
+from .SelfArctan import SelfArctan
+from .LogSigmoid import LogSigmoid
+from .SlopedReLU import SlopedReLU
+from .SmallGaLU import SmallGaLU
+from .MinSin import MinSin
+from .LaLU import LaLU
+from .MexicanReLU import MexicanReLU
+from .APLU import APLU
+from .ERF import ERF
+from .TangentBipolarSigmoidReLU import TangentBipolarSigmoidReLU
+from .BaseDLReLU import BaseDLReLU
+from .Logish import Logish
+from .TripleStateSwish import TripleStateSwish
+from .ExponentialSwish import ExponentialSwish
+from .TeLU import TeLU
+from .Elliot import Elliot
+from .MeLU import MeLU
+from .GaussianReLU import GaussianReLU
+from .ActivationFunction import ActivationFunction
+from .RReLU import RReLU
+from .Suish import Suish
+from .SoftsignRReLU import SoftsignRReLU
+from .Gish import Gish
+from .NReLU import NReLU
+from .LogLogish import LogLogish
+from .SGELU import SGELU
+from .GaLU import GaLU
+from .TSReLU import TSReLU
+from .SineReLU import SineReLU
+from .DLReLU import DLReLU
+from .CaLU import CaLU
+from .RandomizedSlopedReLU import RandomizedSlopedReLU
+from .GCU import GCU
+from .SigmoidDerivative import SigmoidDerivative
+from .Smish import Smish
+from .AdaptiveActivationFunction import AdaptiveActivationFunction
+from .learnable import *

homa/activations/learnable/AOAF.py

@@ -0,0 +1,16 @@
+import torch
+from .concerns import ChannelBased
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+
+
+class AOAF(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self, b: float = 0.17, c: float = 0.17):
+        super().__init__()
+        self.a = None
+        self.b = b
+        self.c = c
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self.initialize(x, "a")
+        a = self.a.view(self.parameter_view(x))
+        return torch.relu(x - self.b * a) + self.c * a

homa/activations/learnable/AReLU.py

@@ -0,0 +1,16 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+
+
+class AReLU(AdaptiveActivationFunction):
+    def __init__(self):
+        super(AReLU, self).__init__()
+        self.a = torch.nn.Parameter(torch.tensor(0.9, requires_grad=True))
+        self.b = torch.nn.Parameter(torch.tensor(2.0, requires_grad=True))
+
+    def forward(self, z):
+        negative_slope = torch.clamp(self.a, 0.01, 0.99)
+        positive_slope = 1 + torch.sigmoid(self.b)
+        positive = positive_slope * torch.relu(z)
+        negative = negative_slope * (-torch.relu(-z))
+        return positive + negative

homa/activations/learnable/DPReLU.py

@@ -0,0 +1,16 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class DPReLU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b"], [1, 0.01])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x, b * x)

homa/activations/learnable/DualLine.py

@@ -0,0 +1,18 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class DualLine(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+        self.m = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b", "m"], [1, 0.01, -0.22])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        m = self.m.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x + m, b * x + m)

homa/activations/learnable/FReLU.py

@@ -0,0 +1,14 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class FReLU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.b = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, "b")
+        b = self.b.view(self.parameter_shape(x))
+        return torch.where(x >= 0, x + b, b)

homa/activations/learnable/LeLeLU.py

@@ -0,0 +1,14 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class LeLeLU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, "a")
+        a = self.a.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x, 0.01 * a * x)

homa/activations/learnable/PERU.py

@@ -0,0 +1,16 @@
+import torch
+from .concerns import ChannelBased
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+
+
+class PERU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b"])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x, a * x * torch.exp(b * x))

homa/activations/learnable/PiLU.py

@@ -0,0 +1,18 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class DualLine(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+        self.c = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b", "c"], [1, 0.01, 1])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        c = self.c.view(self.parameter_shape(x))
+        return torch.where(x >= c, a * x + c * (1 - a), b * x + c * (1 - b))

homa/activations/learnable/ShiLU.py

@@ -0,0 +1,16 @@
+import torch
+from ..ActivationFunction import ActivationFunction
+from .concerns import ChannelBased
+
+
+class ShiLU(ActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self.initialize(x, ["a", "b"])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return torch.relu(x) * a + b

homa/activations/learnable/StarReLU.py

@@ -0,0 +1,16 @@
+import torch
+from ..ActivationFunction import ActivationFunction
+from .concerns import ChannelBased
+
+
+class StarReLU(ActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self.initialize(x, ["a", "b"])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return a * torch.relu(x).pow(2) + b

homa/activations/learnable/__init__.py

@@ -0,0 +1,10 @@
+from .StarReLU import StarReLU
+from .DualLine import DualLine
+from .LeLeLU import LeLeLU
+from .AReLU import AReLU
+from .PERU import PERU
+from .ShiLU import ShiLU
+from .DPReLU import DPReLU
+from .PiLU import DualLine
+from .FReLU import FReLU
+from .AOAF import AOAF

homa/activations/learnable/concerns/ChannelBased.py

@@ -0,0 +1,36 @@
+import torch
+from typing import List
+
+
+class ChannelBased:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._initialized = False
+        self.num_channels = None
+
+    def initialize(
+        self, x: torch.Tensor, attrs: List[str] | str, values: List[float] | float = []
+    ):
+        if getattr(self, "_initialized", False):
+            return
+
+        if not isinstance(values, list):
+            values = [values]
+
+        if not isinstance(attrs, list):
+            attrs = [attrs]
+
+        self.num_channels = x.shape[1]
+        for index, attr in enumerate(attrs):
+            if index < len(values) and values[index] is not None:
+                default_value = float(values[index])
+            else:
+                default_value = 1.0
+            param = torch.nn.Parameter(torch.full((self.num_channels,), default_value))
+            setattr(self, attr, param)
+        self._initialized = True
+
+    def parameter_shape(self, x: torch.Tensor) -> tuple | None:
+        if hasattr(self, "num_channels"):
+            return (1, self.num_channels) + (1,) * (x.ndim - 2)
+        return None

homa/activations/learnable/concerns/__init__.py

@@ -0,0 +1 @@
+from .ChannelBased import ChannelBased

homa/cli/Commands/Command.py

@@ -0,0 +1,2 @@
+class Command:
+    pass

homa/cli/Commands/InitCommand.py

@@ -0,0 +1,34 @@
+import ast
+from pathlib import Path
+from .Command import Command
+
+
+class InitCommand(Command):
+    @staticmethod
+    def run():
+        path = Path(".")
+        init_file = path / "__init__.py"
+        init_file.write_text("")
+        for file in path.iterdir():
+            if file.name == "__init__.py" or file.suffix != ".py":
+                continue
+            tree = ast.parse(file.read_text())
+            classes = [
+                node.name
+                for node in tree.body
+                if isinstance(node, ast.ClassDef) and not node.name.startswith("_")
+            ]
+            functions = [
+                node.name
+                for node in tree.body
+                if isinstance(node, ast.FunctionDef) and not node.name.startswith("_")
+            ]
+            if not (classes or functions):
+                continue
+            module = file.stem
+            lines = [
+                f"from .{module} import {name}\n" for name in (*classes, *functions)
+            ]
+            with init_file.open("a") as f:
+                f.writelines(lines)
+            print(f"Processed {file}: classes={classes}, functions={functions}")

homa/cli/Commands/__init__.py

@@ -0,0 +1,2 @@
+from .Command import Command
+from .InitCommand import InitCommand

homa/cli/HomaCommand.py

@@ -1,5 +1,6 @@
 import fire
 from .namespaces import MakeNamespace, CacheNamespace
+from .Commands import InitCommand
 
 
 class HomaCommand:
@@ -7,6 +8,9 @@ class HomaCommand:
         self.make = MakeNamespace()
         self.cache = CacheNamespace()
 
+    def init(self):
+        InitCommand.run()
+
 
 def main():
     fire.Fire(HomaCommand)

homa/ensemble/Ensemble.py

@@ -1,8 +1,7 @@
 from .concerns import (
     ReportsSize,
-    RecordsStateDictionaries,
+    StoresModels,
     ReportsClassificationMetrics,
-    HasNetwork,
     PredictsProbabilities,
 )
 
@@ -10,9 +9,8 @@ from .concerns import (
 class Ensemble(
     ReportsSize,
     ReportsClassificationMetrics,
-    RecordsStateDictionaries,
     PredictsProbabilities,
-    HasNetwork,
+    StoresModels,
 ):
     def __init__(self):
         super().__init__()

homa/ensemble/concerns/CalculatesMetricNecessities.py

@@ -1,20 +1,24 @@
 import torch
+from ...device import get_device
 
 
 class CalculatesMetricNecessities:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+    @torch.no_grad()
     def metric_necessities(self, dataloader):
-        all_predictions = []
-        all_labels = []
+        predictions, labels = [], []
+        device = get_device()
         for x, y in dataloader:
-            batch_logits_list = []
+            x, y = x.to(device), y.to(device)
+            sum_logits = None
             for model in self.models:
-                batch_logits_list.append(model(x))
-            all_batch_logits = torch.stack(batch_logits_list)
-            avg_logits = torch.mean(all_batch_logits, dim=0)
-            _, preds = torch.max(avg_logits, 1)
-            all_predictions.extend(preds.cpu().numpy())
-            all_labels.extend(y.cpu().numpy())
-        return all_predictions, all_labels
+                model.to(device)
+                model.eval()
+                logits = model(x)
+                sum_logits = logits if sum_logits is None else sum_logits + logits
+            batch_predictions = sum_logits.argmax(dim=1)
+            predictions.extend(batch_predictions.cpu().numpy())
+            labels.extend(y.cpu().numpy())
+        return predictions, labels

homa/ensemble/concerns/PredictsProbabilities.py

@@ -9,3 +9,7 @@ class PredictsProbabilities(ReportsLogits):
     def predict(self, x: torch.Tensor) -> torch.Tensor:
         logits = self.logits(x)
         return torch.nn.functional.softmax(logits, dim=1)
+
+    @torch.no_grad()
+    def predict_(self, x: torch.Tensor) -> torch.Tensor:
+        return self.predict(x)

homa/ensemble/concerns/ReportsClassificationMetrics.py

@@ -6,8 +6,8 @@ from .CalculatesMetricNecessities import CalculatesMetricNecessities
 
 class ReportsClassificationMetrics(
     CalculatesMetricNecessities,
-    ReportsEnsembleF1,
     ReportsEnsembleAccuracy,
+    ReportsEnsembleF1,
     ReportsEnsembleKappa,
 ):
     pass

homa/ensemble/concerns/ReportsEnsembleAccuracy.py

@@ -1,10 +1,11 @@
 from sklearn.metrics import accuracy_score as accuracy
+from torch.utils.data import DataLoader
 
 
 class ReportsEnsembleAccuracy:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
-    def accuracy(self) -> float:
-        predictions, labels = self.metric_necessities()
+    def accuracy(self, dataloader: DataLoader) -> float:
+        predictions, labels = self.metric_necessities(dataloader)
         return accuracy(labels, predictions)

homa/ensemble/concerns/ReportsLogits.py

@@ -11,3 +11,7 @@ class ReportsLogits:
         for model in self.models:
             logits += model(x)
         return logits
+
+    @torch.no_grad()
+    def logits_(self, *args, **kwargs):
+        return self.logits(*args, **kwargs)

homa/ensemble/concerns/ReportsSize.py

@@ -4,8 +4,8 @@ class ReportsSize:
 
     @property
     def size(self):
-        return len(self.state_dicts)
+        return len(self.models)
 
     @property
     def length(self):
-        return len(self.state_dicts)
+        return len(self.models)

homa/ensemble/concerns/StoresModels.py

@@ -0,0 +1,29 @@
+import torch
+from copy import deepcopy
+from typing import List
+from ...vision import Model
+
+
+class StoresModels:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.models: List[torch.nn.Module] = []
+
+    def record(self, model: Model | torch.nn.Module):
+        model_: torch.nn.Module | None = None
+        if isinstance(model, Model):
+            model_ = deepcopy(model.network)
+        elif isinstance(model, torch.nn.Module):
+            model_ = deepcopy(model)
+        else:
+            raise TypeError("Wrong input to ensemble record")
+        self.models.append(model_)
+
+    def push(self, *args, **kwargs):
+        self.record(*args, **kwargs)
+
+    def append(self, *args, **kwargs):
+        self.record(*args, **kwargs)
+
+    def add(self, *args, **kwargs):
+        self.record(*args, **kwargs)

homa/ensemble/concerns/__init__.py

@@ -1,10 +1,9 @@
 from .CalculatesMetricNecessities import CalculatesMetricNecessities
-from .HasNetwork import HasNetwork
 from .PredictsProbabilities import PredictsProbabilities
-from .RecordsStateDictionaries import RecordsStateDictionaries
 from .ReportsClassificationMetrics import ReportsClassificationMetrics
 from .ReportsEnsembleAccuracy import ReportsEnsembleAccuracy
 from .ReportsEnsembleF1 import ReportsEnsembleF1
 from .ReportsEnsembleKappa import ReportsEnsembleKappa
 from .ReportsLogits import ReportsLogits
 from .ReportsSize import ReportsSize
+from .StoresModels import StoresModels

homa/loss/LogitNormLoss.py

@@ -0,0 +1,12 @@
+import torch
+from .Loss import Loss
+
+
+class LogitNormLoss(Loss):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def forward(self, logits, target):
+        norms = torch.norm(logits, p=2, dim=-1, keepdim=True) + 1e-7
+        normalized_logits = torch.div(logits, norms)
+        return torch.nn.functional.cross_entropy(normalized_logits, target)

homa/loss/Loss.py

@@ -0,0 +1,2 @@
+class Loss:
+    pass

homa/loss/__init__.py

@@ -0,0 +1,2 @@
+from .LogitNormLoss import LogitNormLoss
+from .Loss import Loss

homa/torch/__init__.py

@@ -1,2 +1 @@
-from .Module import Module
 from .helpers import *

homa/vision/Classifier.py

@@ -0,0 +1,5 @@
+from .Model import Model
+
+
+class Classifier(Model):
+    pass

homa/vision/Resnet.py

@@ -1,12 +1,13 @@
 import torch
 from .modules import ResnetModule
-from .Model import Model
-from .concerns import Trainable
+from .Classifier import Classifier
+from .concerns import Trainable, ReportsMetrics
+from ..device import get_device
 
 
-class Resnet(Model, Trainable):
-    def __init__(self, num_classes: int, lr: float):
+class Resnet(Classifier, Trainable, ReportsMetrics):
+    def __init__(self, num_classes: int, lr: float = 0.001):
         super().__init__()
-        self.network = ResnetModule(num_classes)
+        self.network = ResnetModule(num_classes).to(get_device())
         self.criterion = torch.nn.CrossEntropyLoss()
         self.optimizer = torch.optim.SGD(self.network.parameters(), lr=lr, momentum=0.9)