homa 0.2.0__py3-none-any.whl → 0.2.95__py3-none-any.whl

homa/activations/TBSReLU.py

@@ -0,0 +1,13 @@
+import torch
+from .ActivationFunction import ActivationFunction
+
+
+class TBSReLU(ActivationFunction):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        a = 1 - torch.exp(-x)
+        b = 1 + torch.exp(-x)
+        c = a / b
+        return x * torch.tanh(c)

homa/activations/TSReLU.py

@@ -0,0 +1,10 @@
+import torch
+from .ActivationFunction import ActivationFunction
+
+
+class TSReLU(ActivationFunction):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return x * torch.tanh(torch.sigmoid(x))

homa/activations/TangentBipolarSigmoidReLU.py

@@ -0,0 +1,6 @@
+from .TBSReLU import TBSReLU
+
+
+class TangentBipolarSigmoidReLU(TBSReLU):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)

homa/activations/TangentSigmoidReLU.py

@@ -0,0 +1,6 @@
+from .TSReLU import TSReLU
+
+
+class TangentSigmoidReLU(TSReLU):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)

homa/activations/TeLU.py

@@ -0,0 +1,9 @@
+import torch
+
+
+class TeLU(torch.nn.Module):
+    def __init__(self):
+        super().__init__()
+
+    def forward(self, x: torch.Tensor):
+        return x * torch.tanh(torch.exp(x))

homa/activations/TripleStateSwish.py

@@ -0,0 +1,15 @@
+import torch
+from .ActivationFunction import ActivationFunction
+
+
+class TripleStateSwish(ActivationFunction):
+    def __init__(self, alpha: float = 20, beta: float = 40, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.alpha = alpha
+        self.beta = beta
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        a = 1 / (1 + torch.exp(-x))
+        b = 1 / (1 + torch.exp(-x + self.alpha))
+        c = 1 / (1 + torch.exp(-x + self.beta))
+        return x * a * (a + b + c)

homa/activations/WideMeLU.py

@@ -0,0 +1,15 @@
+from .MexicanReLU import MexicanReLU
+
+
+class WideMeLU(MexicanReLU):
+    def __init__(self, channels: int | None = None, max_input: float = 1.0):
+        self.hats = [
+            (2.0, 2.0),
+            (1.0, 1.0),
+            (3.0, 1.0),
+            (0.5, 0.5),
+            (1.5, 0.5),
+            (2.5, 0.5),
+            (3.5, 0.5),
+        ]
+        super().__init__(self.hats, channels=channels, max_input=max_input)

homa/activations/__init__.py

@@ -1,2 +1,49 @@
-from .classes import *
-from .utils import *
+from .ShiftedReLU import ShiftedReLU
+from .PLogish import PLogish
+from .ParametricLogish import ParametricLogish
+from .ExpExpish import ExpExpish
+from .GeneralizedSwish import GeneralizedSwish
+from .TBSReLU import TBSReLU
+from .NoisyReLU import NoisyReLU
+from .ExponentialDLReLU import ExponentialDLReLU
+from .SReLU import SReLU
+from .TangentSigmoidReLU import TangentSigmoidReLU
+from .Phish import Phish
+from .WideMeLU import WideMeLU
+from .SelfArctan import SelfArctan
+from .LogSigmoid import LogSigmoid
+from .SlopedReLU import SlopedReLU
+from .SmallGaLU import SmallGaLU
+from .MinSin import MinSin
+from .LaLU import LaLU
+from .MexicanReLU import MexicanReLU
+from .APLU import APLU
+from .ERF import ERF
+from .TangentBipolarSigmoidReLU import TangentBipolarSigmoidReLU
+from .BaseDLReLU import BaseDLReLU
+from .Logish import Logish
+from .TripleStateSwish import TripleStateSwish
+from .ExponentialSwish import ExponentialSwish
+from .TeLU import TeLU
+from .Elliot import Elliot
+from .MeLU import MeLU
+from .GaussianReLU import GaussianReLU
+from .ActivationFunction import ActivationFunction
+from .RReLU import RReLU
+from .Suish import Suish
+from .SoftsignRReLU import SoftsignRReLU
+from .Gish import Gish
+from .NReLU import NReLU
+from .LogLogish import LogLogish
+from .SGELU import SGELU
+from .GaLU import GaLU
+from .TSReLU import TSReLU
+from .SineReLU import SineReLU
+from .DLReLU import DLReLU
+from .CaLU import CaLU
+from .RandomizedSlopedReLU import RandomizedSlopedReLU
+from .GCU import GCU
+from .SigmoidDerivative import SigmoidDerivative
+from .Smish import Smish
+from .AdaptiveActivationFunction import AdaptiveActivationFunction
+from .learnable import *

homa/activations/learnable/AOAF.py

@@ -0,0 +1,16 @@
+import torch
+from .concerns import ChannelBased
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+
+
+class AOAF(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self, b: float = 0.17, c: float = 0.17):
+        super().__init__()
+        self.a = None
+        self.b = b
+        self.c = c
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self.initialize(x, "a")
+        a = self.a.view(self.parameter_shape(x))
+        return torch.relu(x - self.b * a) + self.c * a

homa/activations/learnable/AReLU.py

@@ -0,0 +1,19 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from ...device import get_device
+
+
+class AReLU(AdaptiveActivationFunction):
+    def __init__(self):
+        super(AReLU, self).__init__()
+        self.a = torch.nn.Parameter(torch.tensor(0.9, requires_grad=True))
+        self.b = torch.nn.Parameter(torch.tensor(2.0, requires_grad=True))
+        self.a.to(get_device())
+        self.b.to(get_device())
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        negative_slope = torch.clamp(self.a, 0.01, 0.99)
+        positive_slope = 1 + torch.sigmoid(self.b)
+        positive = positive_slope * torch.relu(x)
+        negative = negative_slope * (-torch.relu(-x))
+        return positive + negative

homa/activations/learnable/DPReLU.py

@@ -0,0 +1,16 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class DPReLU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b"], [1, 0.01])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x, b * x)

homa/activations/learnable/DualLine.py

@@ -0,0 +1,18 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class DualLine(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+        self.m = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b", "m"], [1, 0.01, -0.22])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        m = self.m.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x + m, b * x + m)

homa/activations/learnable/FReLU.py

@@ -0,0 +1,14 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class FReLU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.b = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, "b")
+        b = self.b.view(self.parameter_shape(x))
+        return torch.where(x >= 0, x + b, b)

homa/activations/learnable/LeLeLU.py

@@ -0,0 +1,14 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class LeLeLU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, "a")
+        a = self.a.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x, 0.01 * a * x)

homa/activations/learnable/PERU.py

@@ -0,0 +1,16 @@
+import torch
+from .concerns import ChannelBased
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+
+
+class PERU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b"])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return torch.where(x >= 0, a * x, a * x * torch.exp(b * x))

homa/activations/learnable/PiLU.py

@@ -0,0 +1,18 @@
+import torch
+from ..AdaptiveActivationFunction import AdaptiveActivationFunction
+from .concerns import ChannelBased
+
+
+class PiLU(AdaptiveActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+        self.c = None
+
+    def forward(self, x: torch.Tensor):
+        self.initialize(x, ["a", "b", "c"], [1, 0.01, 1])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        c = self.c.view(self.parameter_shape(x))
+        return torch.where(x >= c, a * x + c * (1 - a), b * x + c * (1 - b))

homa/activations/learnable/ShiLU.py

@@ -0,0 +1,16 @@
+import torch
+from ..ActivationFunction import ActivationFunction
+from .concerns import ChannelBased
+
+
+class ShiLU(ActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self.initialize(x, ["a", "b"])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return torch.relu(x) * a + b

homa/activations/learnable/StarReLU.py

@@ -0,0 +1,16 @@
+import torch
+from ..ActivationFunction import ActivationFunction
+from .concerns import ChannelBased
+
+
+class StarReLU(ActivationFunction, ChannelBased):
+    def __init__(self):
+        super().__init__()
+        self.a = None
+        self.b = None
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        self.initialize(x, ["a", "b"])
+        a = self.a.view(self.parameter_shape(x))
+        b = self.b.view(self.parameter_shape(x))
+        return a * torch.relu(x).pow(2) + b

homa/activations/learnable/__init__.py

@@ -0,0 +1,10 @@
+from .DualLine import DualLine
+from .LeLeLU import LeLeLU
+from .AReLU import AReLU
+from .PERU import PERU
+from .ShiLU import ShiLU
+from .StarReLU import StarReLU
+from .DPReLU import DPReLU
+from .PiLU import PiLU
+from .FReLU import FReLU
+from .AOAF import AOAF

homa/activations/learnable/concerns/ChannelBased.py

@@ -0,0 +1,38 @@
+import torch
+from typing import List
+
+
+class ChannelBased:
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._initialized = False
+        self.num_channels = None
+
+    def initialize(
+        self, x: torch.Tensor, attrs: List[str] | str, values: List[float] | float = []
+    ):
+        if getattr(self, "_initialized", False):
+            return
+
+        if not isinstance(values, list):
+            values = [values]
+
+        if not isinstance(attrs, list):
+            attrs = [attrs]
+
+        self.num_channels = x.shape[1]
+        device = x.device
+        for index, attr in enumerate(attrs):
+            if index < len(values) and values[index] is not None:
+                default_value = float(values[index])
+            else:
+                default_value = 1.0
+            param = torch.nn.Parameter(torch.full((self.num_channels,), default_value))
+            param = param.to(device)
+            setattr(self, attr, param)
+        self._initialized = True
+
+    def parameter_shape(self, x: torch.Tensor) -> tuple | None:
+        if hasattr(self, "num_channels"):
+            return (1, self.num_channels) + (1,) * (x.ndim - 2)
+        return None

homa/activations/learnable/concerns/__init__.py

@@ -0,0 +1 @@
+from .ChannelBased import ChannelBased

homa/cli/Commands/Command.py

@@ -0,0 +1,2 @@
+class Command:
+    pass

homa/cli/Commands/InitCommand.py

@@ -0,0 +1,34 @@
+import ast
+from pathlib import Path
+from .Command import Command
+
+
+class InitCommand(Command):
+    @staticmethod
+    def run():
+        path = Path(".")
+        init_file = path / "__init__.py"
+        init_file.write_text("")
+        for file in path.iterdir():
+            if file.name == "__init__.py" or file.suffix != ".py":
+                continue
+            tree = ast.parse(file.read_text())
+            classes = [
+                node.name
+                for node in tree.body
+                if isinstance(node, ast.ClassDef) and not node.name.startswith("_")
+            ]
+            functions = [
+                node.name
+                for node in tree.body
+                if isinstance(node, ast.FunctionDef) and not node.name.startswith("_")
+            ]
+            if not (classes or functions):
+                continue
+            module = file.stem
+            lines = [
+                f"from .{module} import {name}\n" for name in (*classes, *functions)
+            ]
+            with init_file.open("a") as f:
+                f.writelines(lines)
+            print(f"Processed {file}: classes={classes}, functions={functions}")

homa/cli/Commands/__init__.py

@@ -0,0 +1,2 @@
+from .Command import Command
+from .InitCommand import InitCommand

homa/cli/HomaCommand.py

@@ -1,5 +1,6 @@
 import fire
 from .namespaces import MakeNamespace, CacheNamespace
+from .Commands import InitCommand
 
 
 class HomaCommand:
@@ -7,6 +8,9 @@ class HomaCommand:
         self.make = MakeNamespace()
         self.cache = CacheNamespace()
 
+    def init(self):
+        InitCommand.run()
+
 
 def main():
     fire.Fire(HomaCommand)

homa/ensemble/concerns/StoresModels.py

@@ -1,5 +1,5 @@
 import torch
-from copy import deepcopy
+import io
 from typing import List
 from ...vision import Model
 
@@ -12,11 +12,17 @@ class StoresModels:
     def record(self, model: Model | torch.nn.Module):
         model_: torch.nn.Module | None = None
         if isinstance(model, Model):
-            model_ = deepcopy(model.network)
+            model_ = model.network
         elif isinstance(model, torch.nn.Module):
-            model_ = deepcopy(model)
+            model_ = model
         else:
             raise TypeError("Wrong input to ensemble record")
+
+        device = model_.device
+        buffer = io.BytesIO()
+        torch.save(model_.to("cpu"), buffer)
+        buffer.seek(0)
+        model_ = torch.load(buffer, map_location=device)
         self.models.append(model_)
 
     def push(self, *args, **kwargs):

homa/vision/{ClassificationModel.py → Classifier.py}

@@ -1,5 +1,5 @@
 from .Model import Model
 
 
-class ClassificationModel(Model):
+class Classifier(Model):
     pass

homa/vision/Resnet.py

@@ -1,11 +1,11 @@
 import torch
 from .modules import ResnetModule
-from .ClassificationModel import ClassificationModel
+from .Classifier import Classifier
 from .concerns import Trainable, ReportsMetrics
 from ..device import get_device
 
 
-class Resnet(ClassificationModel, Trainable, ReportsMetrics):
+class Resnet(Classifier, Trainable, ReportsMetrics):
     def __init__(self, num_classes: int, lr: float = 0.001):
         super().__init__()
         self.network = ResnetModule(num_classes).to(get_device())

homa/vision/StochasticClassifier.py

@@ -1,28 +1,29 @@
-import torch
-import random
-from ..activations import APLU, GALU, SmallGALU, MELU, WideMELU, PDELU, SReLU
+from ..activations import (
+    AOAF,
+    AReLU,
+    DPReLU,
+    DualLine,
+    FReLU,
+    LeLeLU,
+    PERU,
+    PiLU,
+    ShiLU,
+    StarReLU,
+)
 
 
 class StochasticClassifier:
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.pool = [
-            APLU,
-            GALU,
-            SmallGALU,
-            MELU,
-            WideMELU,
-            PDELU,
-            SReLU,
-            torch.nn.ReLU,
-            torch.nn.PReLU,
-            torch.nn.LeakyReLU,
-            torch.nn.ELU,
+        self._activation_pool = [
+            AOAF,
+            AReLU,
+            DPReLU,
+            DualLine,
+            FReLU,
+            LeLeLU,
+            PERU,
+            PiLU,
+            ShiLU,
+            StarReLU,
         ]
-
-    def replace_activations(self, needle: torch.Tensor) -> None:
-        replacement = random.choice(self.pool)
-        for parent in self.network.modules():
-            for name, child in list(parent.named_children()):
-                if isinstance(child, needle):
-                    setattr(parent, name, replacement())

homa/vision/StochasticSwin.py

@@ -1,9 +1,11 @@
 import torch
 from .Swin import Swin
 from .StochasticClassifier import StochasticClassifier
+from .utils import replace_activations
 
 
 class StochasticSwin(Swin, StochasticClassifier):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.replace_activations(torch.nn.GELU)
+        replace_activations(self.network, torch.nn.GELU, self._activation_pool)
+        replace_activations(self.network, torch.nn.ReLU, self._activation_pool)

homa/vision/Swin.py

@@ -1,12 +1,13 @@
 import torch
-from .ClassificationModel import ClassificationModel
+from .Classifier import Classifier
 from .concerns import Trainable, ReportsMetrics
 from .modules import SwinModule
+from ..device import get_device
 
 
-class Swin(ClassificationModel, Trainable, ReportsMetrics):
+class Swin(Classifier, Trainable, ReportsMetrics):
     def __init__(self, num_classes: int, lr: float = 0.0001):
         super().__init__()
-        self.network = SwinModule(num_classes=num_classes)
+        self.network = SwinModule(num_classes=num_classes).to(get_device())
         self.optimizer = torch.optim.AdamW(self.network.parameters(), lr=lr)
         self.criterion = torch.nn.CrossEntropyLoss()

homa/vision/__init__.py

@@ -1,4 +1,5 @@
+from .Classifier import Classifier
 from .Model import Model
 from .Resnet import Resnet
-from .StochasticResnet import StochasticResnet
 from .Swin import Swin
+from .StochasticSwin import StochasticSwin

homa/vision/utils.py

@@ -0,0 +1,12 @@
+import torch
+import random
+
+
+def replace_activations(module, needle: torch.nn.Module, candidates: list):
+    for name, child in module.named_children():
+        if isinstance(child, needle):
+            new_activation = random.choice(candidates)
+            setattr(module, name, new_activation())
+        else:
+            replace_activations(child, needle, candidates)
+    return module

{homa-0.2.0.dist-info → homa-0.2.95.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: homa
-Version: 0.2.0
+Version: 0.2.95
 Summary: A curated list of machine learning and deep learning helpers.
 Author-email: Taha Shieenavaz <tahashieenavaz@gmail.com>
 Requires-Python: >=3.7