lt-tensor 0.0.1a20__tar.gz → 0.0.1a22__tar.gz

{lt_tensor-0.0.1a20 → lt_tensor-0.0.1a22}/LICENSE

@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [2025] [gr1336 (Gabriel Ribeiro)]
+   Copyright 2025 gr1336
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

{lt_tensor-0.0.1a20 → lt_tensor-0.0.1a22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: lt-tensor
-Version: 0.0.1a20
+Version: 0.0.1a22
 Summary: General utilities for PyTorch and others. Built for general use.
 Home-page: https://github.com/gr1336/lt-tensor/
 Author: gr1336
@@ -17,11 +17,12 @@ Requires-Dist: numpy>=1.26.4
 Requires-Dist: tokenizers
 Requires-Dist: pyyaml>=6.0.0
 Requires-Dist: numba>0.60.0
-Requires-Dist: lt-utils==0.0.2a3
+Requires-Dist: lt-utils==0.0.2
 Requires-Dist: librosa==0.11.*
 Requires-Dist: einops
 Requires-Dist: plotly
 Requires-Dist: scipy
+Requires-Dist: huggingface_hub
 Dynamic: author
 Dynamic: classifier
 Dynamic: description

{lt_tensor-0.0.1a20 → lt_tensor-0.0.1a22}/lt_tensor/model_base.py

@@ -194,10 +194,11 @@ class Model(_Devices_Base, ABC):
         None  # Example: lambda: nn.Linear(32, 32, True)
     )
     low_rank_adapter: Union[nn.Identity, nn.Module, nn.Sequential] = nn.Identity()
+    # never freeze:
+    _never_freeze_modules: List[str] = ["low_rank_adapter"]
 
     # dont save list:
     _dont_save_items: List[str] = []
-    _loss_history: LossTracker = LossTracker(20_000)
 
     @property
     def autocast(self):
@@ -207,12 +208,16 @@ class Model(_Devices_Base, ABC):
     def autocast(self, value: bool):
         self._autocast = value
 
-    def freeze_all(self, exclude: Optional[List[str]] = None):
+    def freeze_all(self, exclude: Optional[List[str]] = None, force: bool = False):
         no_exclusions = not exclude
         no_exclusions = not exclude
         results = []
         for name, module in self.named_modules():
-            if name not in self.registered_freezable_modules:
+            if (
+                name in self._never_freeze_modules
+                or not force
+                and name not in self.registered_freezable_modules
+            ):
                 results.append(
                     (
                         name,
@@ -228,12 +233,17 @@ class Model(_Devices_Base, ABC):
                 results.append((name, "excluded"))
         return results
 
-    def unfreeze_all(self, exclude: Optional[list[str]] = None):
+    def unfreeze_all(self, exclude: Optional[list[str]] = None, force: bool = False):
         """Unfreezes all model parameters except specified layers."""
         no_exclusions = not exclude
         results = []
         for name, module in self.named_modules():
-            if name not in self.registered_freezable_modules:
+            if (
+                name in self._never_freeze_modules
+                or not force
+                and name not in self.registered_freezable_modules
+            ):
+
                 results.append(
                     (
                         name,
@@ -250,6 +260,9 @@ class Model(_Devices_Base, ABC):
         return results
 
     def change_frozen_state(self, freeze: bool, module: nn.Module):
+        assert isinstance(module, nn.Module)
+        if module.__class__.__name__ in self._never_freeze_modules:
+            return "Not Allowed"
         try:
             if isinstance(module, Model):
                 if module._can_be_frozen:
@@ -258,7 +271,7 @@ class Model(_Devices_Base, ABC):
                     return module.unfreeze_all()
                 else:
                     return "Not Allowed"
-            elif isinstance(module, nn.Module):
+            else:
                 module.requires_grad_(not freeze)
                 return not freeze
         except Exception as e:
@@ -451,7 +464,7 @@ class Model(_Devices_Base, ABC):
         raise_if_not_exists: bool = False,
         strict: bool = False,
         assign: bool = False,
-        weights_only: bool = True,
+        weights_only: bool = False,
         mmap: Optional[bool] = None,
         **pickle_load_args,
     ):
@@ -505,37 +518,3 @@ class Model(_Devices_Base, ABC):
         self, *args, **kwargs
     ) -> Union[Tensor, Sequence[Tensor], Dict[Any, Union[Any, Tensor]]]:
         pass
-
-    def add_loss(
-        self,
-        loss: Union[float, list[float]],
-        mode: Union[Literal["train", "eval"]] = "train",
-    ):
-        if isinstance(loss, Number) and loss:
-            self._loss_history.append(loss, mode)
-        elif isinstance(loss, (list, tuple)):
-            if loss:
-                self._loss_history.append(sum(loss) / len(loss), mode=mode)
-        elif isinstance(loss, Tensor):
-            try:
-                self._loss_history.append(loss.detach().flatten().mean().item())
-            except Exception as e:
-                log_traceback(e, "add_loss - Tensor")
-
-    def save_loss_history(self, path: Optional[PathLike] = None):
-        self._loss_history.save(path)
-
-    def load_loss_history(self, path: Optional[PathLike] = None):
-        self._loss_history.load(path)
-
-    def get_loss_avg(
-        self,
-        mode: Union[Literal["train", "eval"], str],
-        quantity: int = 0,
-    ):
-        t_list = self._loss_history.get(mode)
-        if not t_list:
-            return float("nan")
-        if quantity > 0:
-            t_list = t_list[-quantity:]
-        return sum(t_list) / len(t_list)

lt_tensor-0.0.1a22/lt_tensor/model_zoo/activations/alias_free_torch/__init__.py

@@ -0,0 +1 @@
+from . import *

lt_tensor-0.0.1a22/lt_tensor/model_zoo/activations/alias_free_torch/act.py

@@ -0,0 +1,55 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .resample import UpSample1d, DownSample1d
+from .resample import UpSample2d, DownSample2d
+
+
+class Activation1d(nn.Module):
+
+    def __init__(
+        self,
+        activation,
+        up_ratio: int = 2,
+        down_ratio: int = 2,
+        up_kernel_size: int = 12,
+        down_kernel_size: int = 12,
+    ):
+        super().__init__()
+        self.up_ratio = up_ratio
+        self.down_ratio = down_ratio
+        self.act = activation
+        self.upsample = UpSample1d(up_ratio, up_kernel_size)
+        self.downsample = DownSample1d(down_ratio, down_kernel_size)
+
+    # x: [B,C,T]
+    def forward(self, x):
+        x = self.upsample(x)
+        x = self.act(x)
+        x = self.downsample(x)
+        return x
+
+
+class Activation2d(nn.Module):
+
+    def __init__(
+        self,
+        activation,
+        up_ratio: int = 2,
+        down_ratio: int = 2,
+        up_kernel_size: int = 12,
+        down_kernel_size: int = 12,
+    ):
+        super().__init__()
+        self.up_ratio = up_ratio
+        self.down_ratio = down_ratio
+        self.act = activation
+        self.upsample = UpSample2d(up_ratio, up_kernel_size)
+        self.downsample = DownSample2d(down_ratio, down_kernel_size)
+
+    # x: [B,C,W,H]
+    def forward(self, x):
+        x = self.upsample(x)
+        x = self.act(x)
+        x = self.downsample(x)
+        return x

lt_tensor-0.0.1a22/lt_tensor/model_zoo/activations/alias_free_torch/filter.py

@@ -0,0 +1,183 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import math
+
+if "sinc" in dir(torch):
+    sinc = torch.sinc
+else:
+    # This code is adopted from adefossez's julius.core.sinc
+    # https://adefossez.github.io/julius/julius/core.html
+    def sinc(x: torch.Tensor):
+        """
+        Implementation of sinc, i.e. sin(pi * x) / (pi * x)
+        __Warning__: Different to julius.sinc, the input is multiplied by `pi`!
+        """
+        return torch.where(
+            x == 0,
+            torch.tensor(1.0, device=x.device, dtype=x.dtype),
+            torch.sin(math.pi * x) / math.pi / x,
+        )
+
+
+# This code is adopted from adefossez's julius.lowpass.LowPassFilters
+# https://adefossez.github.io/julius/julius/lowpass.html
+
+
+# return filter [1,1,kernel_size]
+def kaiser_sinc_filter1d(cutoff, half_width, kernel_size):
+    even = kernel_size % 2 == 0
+    half_size = kernel_size // 2
+
+    # For kaiser window
+    delta_f = 4 * half_width
+    A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95
+    if A > 50.0:
+        beta = 0.1102 * (A - 8.7)
+    elif A >= 21.0:
+        beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0)
+    else:
+        beta = 0.0
+    window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)
+    # ratio = 0.5/cutoff -> 2 * cutoff = 1 / ratio
+    if even:
+        time = torch.arange(-half_size, half_size) + 0.5
+    else:
+        time = torch.arange(kernel_size) - half_size
+    if cutoff == 0:
+        filter_ = torch.zeros_like(time)
+    else:
+        filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)
+        # Normalize filter to have sum = 1, otherwise we will have a small leakage
+        # of the constant component in the input signal.
+        filter_ /= filter_.sum()
+        filter = filter_.view(1, 1, kernel_size)
+    return filter
+
+
+def kaiser_sinc_filter2d(cutoff, half_width, kernel_size):
+    even = kernel_size % 2 == 0
+    half_size = kernel_size // 2
+    # For kaiser window
+    delta_f = 4 * half_width
+    A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95
+    if A > 50.0:
+        beta = 0.1102 * (A - 8.7)
+    elif A >= 21.0:
+        beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0)
+    else:
+        beta = 0.0
+
+    # rotation equivariant grid
+    if even:
+        time = torch.stack(
+            torch.meshgrid(
+                torch.arange(-half_size, half_size) + 0.5,
+                torch.arange(-half_size, half_size) + 0.5,
+            ),
+            dim=-1,
+        )
+    else:
+        time = torch.stack(
+            torch.meshgrid(
+                torch.arange(kernel_size) - half_size,
+                torch.arange(kernel_size) - half_size,
+            ),
+            dim=-1,
+        )
+
+    time = torch.norm(time, dim=-1)
+    # rotation equivariant window
+    window = torch.i0(
+        beta * torch.sqrt(1 - (time / half_size / 2**0.5) ** 2)
+    ) / torch.i0(torch.tensor([beta]))
+    # ratio = 0.5/cutroff
+    if cutoff == 0:
+        filter_ = torch.zeros_like(time)
+    else:
+        filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)
+        # Normalize filter to have sum = 1, otherwise we will have a small leakage
+        # of the constant component in the input signal.
+        filter_ /= filter_.sum()
+        filter = filter_.view(1, 1, kernel_size, kernel_size)
+    return filter
+
+
+class LowPassFilter1d(nn.Module):
+
+    def __init__(
+        self,
+        cutoff=0.5,
+        half_width=0.6,
+        stride: int = 1,
+        padding: bool = True,
+        padding_mode: str = "replicate",
+        kernel_size: int = 12,
+    ):
+        # kernel_size should be even number for stylegan3 setup,
+        # in this implementation, odd number is also possible.
+        super().__init__()
+        if cutoff < -0.0:
+            raise ValueError("Minimum cutoff must be larger than zero.")
+        if cutoff > 0.5:
+            raise ValueError("A cutoff above 0.5 does not make sense.")
+        self.kernel_size = kernel_size
+        self.even = kernel_size % 2 == 0
+        self.pad_left = kernel_size // 2 - int(self.even)
+        self.pad_right = kernel_size // 2
+        self.stride = stride
+        self.padding = padding
+        self.padding_mode = padding_mode
+        filter = kaiser_sinc_filter1d(cutoff, half_width, kernel_size)
+        self.register_buffer("filter", filter)
+
+    # input [B,C,T]
+    def forward(self, x):
+        _, C, _ = x.shape
+        if self.padding:
+            x = F.pad(x, (self.pad_left, self.pad_right), mode=self.padding_mode)
+        out = F.conv1d(x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C)
+        return out
+
+
+class LowPassFilter2d(nn.Module):
+
+    def __init__(
+        self,
+        cutoff=0.5,
+        half_width=0.6,
+        stride: int = 1,
+        padding: bool = True,
+        padding_mode: str = "replicate",
+        kernel_size: int = 12,
+    ):
+        # kernel_size should be even number for stylegan3 setup,
+        # in this implementation, odd number is also possible.
+        super().__init__()
+        if cutoff < -0.0:
+            raise ValueError("Minimum cutoff must be larger than zero.")
+        if cutoff > 0.5:
+            raise ValueError("A cutoff above 0.5 does not make sense.")
+        self.kernel_size = kernel_size
+        self.even = kernel_size % 2 == 0
+        self.pad_left = kernel_size // 2 - int(self.even)
+        self.pad_right = kernel_size // 2
+        self.stride = stride
+        self.padding = padding
+        self.padding_mode = padding_mode
+        filter = kaiser_sinc_filter2d(cutoff, half_width, kernel_size)
+        self.register_buffer("filter", filter)
+
+    # input [B,C,W,H]
+    def forward(self, x):
+        _, C, _, _ = x.shape
+        if self.padding:
+            x = F.pad(
+                x,
+                (self.pad_left, self.pad_right, self.pad_left, self.pad_right),
+                mode=self.padding_mode,
+            )
+        out = F.conv2d(
+            x, self.filter.expand(C, -1, -1, -1), stride=self.stride, groups=C
+        )
+        return out

lt_tensor-0.0.1a22/lt_tensor/model_zoo/activations/alias_free_torch/resample.py

@@ -0,0 +1,106 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from .filter import LowPassFilter1d, LowPassFilter2d
+from .filter import kaiser_sinc_filter1d, kaiser_sinc_filter2d
+
+
+class UpSample1d(nn.Module):
+
+    def __init__(self, ratio=2, kernel_size=None):
+        super().__init__()
+        self.ratio = ratio
+        self.kernel_size = (
+            int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+        )
+        self.stride = ratio
+        self.pad = self.kernel_size // ratio - 1
+        self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
+        self.pad_right = (
+            self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
+        )
+        filter = kaiser_sinc_filter1d(
+            cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size
+        )
+        self.register_buffer("filter", filter)
+
+    # x: [B,C,T]
+    def forward(self, x):
+        _, C, _ = x.shape
+        x = F.pad(x, (self.pad, self.pad), mode="replicate")
+        x = self.ratio * F.conv_transpose1d(
+            x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C
+        )
+        x = x[..., self.pad_left : -self.pad_right]
+        return x
+
+
+class DownSample1d(nn.Module):
+
+    def __init__(self, ratio=2, kernel_size=None):
+        super().__init__()
+        self.ratio = ratio
+        self.kernel_size = (
+            int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+        )
+        self.lowpass = LowPassFilter1d(
+            cutoff=0.5 / ratio,
+            half_width=0.6 / ratio,
+            stride=ratio,
+            kernel_size=self.kernel_size,
+        )
+
+    def forward(self, x):
+        xx = self.lowpass(x)
+        return xx
+
+
+class UpSample2d(nn.Module):
+
+    def __init__(self, ratio=2, kernel_size=None):
+        super().__init__()
+        self.ratio = ratio
+        self.kernel_size = (
+            int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+        )
+        self.stride = ratio
+        self.pad = self.kernel_size // 2 - ratio // 2
+        self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2
+        self.pad_right = (
+            self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2
+        )
+        filter = kaiser_sinc_filter2d(
+            cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=self.kernel_size
+        )
+        self.register_buffer("filter", filter)
+
+    # x: [B,C,W,H]
+    def forward(self, x):
+        _, C, _, _ = x.shape
+        x = F.pad(x, (self.pad, self.pad, self.pad, self.pad), mode="replicate")
+        x = self.ratio**2 * F.conv_transpose2d(
+            x, self.filter.expand(C, -1, -1, -1), stride=self.stride, groups=C
+        )
+        x = x[..., self.pad_left : -self.pad_right, self.pad_left : -self.pad_right]
+        return x
+
+
+class DownSample2d(nn.Module):
+
+    def __init__(self, ratio=2, kernel_size=None):
+        super().__init__()
+        self.ratio = ratio
+        self.kernel_size = (
+            int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size
+        )
+        self.lowpass = LowPassFilter2d(
+            cutoff=0.5 / ratio,
+            half_width=0.6 / ratio,
+            stride=ratio,
+            kernel_size=self.kernel_size,
+        )
+
+    # x: [B,C,W,H]
+    def forward(self, x):
+        xx = self.lowpass(x)
+        return xx

lt_tensor-0.0.1a22/lt_tensor/model_zoo/activations/snake/__init__.py

@@ -0,0 +1,129 @@
+# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license.
+
+import torch
+from torch import nn, sin, pow
+from torch.nn import Parameter
+
+
+class Snake(nn.Module):
+    """
+    Implementation of a sine-based periodic activation function
+    Shape:
+        - Input: (B, C, T)
+        - Output: (B, C, T), same shape as the input
+    Parameters:
+        - alpha - trainable parameter
+    References:
+        - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
+        https://arxiv.org/abs/2006.08195
+    Examples:
+        >>> a1 = snake(256)
+        >>> x = torch.randn(256)
+        >>> x = a1(x)
+    """
+
+    def __init__(
+        self,
+        in_features,
+        alpha=1.0,
+        alpha_trainable=True,
+        alpha_logscale=False,
+    ):
+        """
+        Initialization.
+        INPUT:
+            - in_features: shape of the input
+            - alpha: trainable parameter
+            alpha is initialized to 1 by default, higher values = higher-frequency.
+            alpha will be trained along with the rest of your model.
+        """
+        super(Snake, self).__init__()
+        self.in_features = in_features
+
+        # initialize alpha
+        self.alpha_logscale = alpha_logscale
+        if self.alpha_logscale:  # log scale alphas initialized to zeros
+            self.alpha = Parameter(torch.zeros(in_features) * alpha)
+        else:  # linear scale alphas initialized to ones
+            self.alpha = Parameter(torch.ones(in_features) * alpha)
+
+        self.alpha.requires_grad = alpha_trainable
+
+        self.no_div_by_zero = 1e-7
+
+    def forward(self, x):
+        """
+        Forward pass of the function.
+        Applies the function to the input elementwise.
+        Snake ∶= x + 1/a * sin^2 (xa)
+        """
+        alpha = self.alpha.unsqueeze(0).unsqueeze(-1)  # line up with x to [B, C, T]
+        if self.alpha_logscale:
+            alpha = torch.exp(alpha)
+        x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2)
+
+        return x
+
+
+class SnakeBeta(nn.Module):
+    """
+    A modified Snake function which uses separate parameters for the magnitude of the periodic components
+    Shape:
+        - Input: (B, C, T)
+        - Output: (B, C, T), same shape as the input
+    Parameters:
+        - alpha - trainable parameter that controls frequency
+        - beta - trainable parameter that controls magnitude
+    References:
+        - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:
+        https://arxiv.org/abs/2006.08195
+    Examples:
+        >>> a1 = snakebeta(256)
+        >>> x = torch.randn(256)
+        >>> x = a1(x)
+    """
+
+    def __init__(
+        self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False
+    ):
+        """
+        Initialization.
+        INPUT:
+            - in_features: shape of the input
+            - alpha - trainable parameter that controls frequency
+            - beta - trainable parameter that controls magnitude
+            alpha is initialized to 1 by default, higher values = higher-frequency.
+            beta is initialized to 1 by default, higher values = higher-magnitude.
+            alpha will be trained along with the rest of your model.
+        """
+        super(SnakeBeta, self).__init__()
+        self.in_features = in_features
+
+        # initialize alpha
+        self.alpha_logscale = alpha_logscale
+        if self.alpha_logscale:  # log scale alphas initialized to zeros
+            self.alpha = Parameter(torch.zeros(in_features) * alpha)
+            self.beta = Parameter(torch.zeros(in_features) * alpha)
+        else:  # linear scale alphas initialized to ones
+            self.alpha = Parameter(torch.ones(in_features) * alpha)
+            self.beta = Parameter(torch.ones(in_features) * alpha)
+
+        self.alpha.requires_grad = alpha_trainable
+        self.beta.requires_grad = alpha_trainable
+
+        self.no_div_by_zero = 1e-7
+
+    def forward(self, x):
+        """
+        Forward pass of the function.
+        Applies the function to the input elementwise.
+        SnakeBeta ∶= x + 1/b * sin^2 (xa)
+        """
+        alpha = self.alpha.unsqueeze(0).unsqueeze(-1)  # line up with x to [B, C, T]
+        beta = self.beta.unsqueeze(0).unsqueeze(-1)
+        if self.alpha_logscale:
+            alpha = torch.exp(alpha)
+            beta = torch.exp(beta)
+        x = x + (1.0 / (beta + self.no_div_by_zero)) * pow(sin(x * alpha), 2)
+
+        return x