lt-tensor 0.0.1a10__py3-none-any.whl → 0.0.1a12__py3-none-any.whl

lt_tensor/__init__.py

@@ -12,6 +12,7 @@ from . import (
     losses,
     processors,
     datasets,
+    torch_commons,
 )
 
 __all__ = [
@@ -26,4 +27,5 @@ __all__ = [
     "losses",
     "processors",
     "datasets",
+    "torch_commons",
 ]

lt_tensor/config_templates.py

@@ -0,0 +1,97 @@
+from lt_utils.common import *
+from lt_utils.file_ops import load_json, save_json, FileScan
+from lt_utils.misc_utils import log_traceback, get_current_time
+from lt_utils.type_utils import is_pathlike, is_file, is_dir, is_dict, is_str
+from lt_tensor.misc_utils import updateDict
+
+
+class ModelConfig(ABC, OrderedDict):
+    _default_settings: Dict[str, Any] = {}
+    _forbidden_list: List[str] = [
+        "_settings",
+    ]
+
+    def __init__(
+        self,
+        settings: Dict[str, Any] = None,
+        path_name: Optional[Union[str, PathLike]] = None,
+    ):
+        assert is_dict(settings)
+        self._default_settings = settings
+        if path_name is not None and is_pathlike(path_name):
+            if not str(path_name).endswith(".json"):
+                self.path_name = str(Path(path_name, "config.json")).replace("\\", "/")
+            else:
+                self.path_name = str(path_name).replace("\\", "/")
+        else:
+            self.path_name = "config.json"
+        self.reset_settings()
+
+    def _setup_path_name(self, path_name: Union[str, PathLike]):
+        if is_file(path_name):
+            self.from_path(path_name)
+            self.path_name = str(path_name).replace("\\", "/")
+        elif is_str(path_name):
+            self.path_name = str(path_name).replace("\\", "/")
+            if not self.path_name.endswith((".json")):
+                self.path_name += ".json"
+
+    def reset_settings(self):
+        for s_name, setting in self._default_settings.items():
+            if s_name in self._forbidden_list:
+                continue
+            updateDict(self, {s_name: setting})
+
+    def save_config(
+        self,
+        path_name: Union[PathLike, str],
+    ):
+        assert is_pathlike(
+            path_name, True
+        ), f"path_name should be a non-empty string or pathlike object! received instead: {path_name}"
+        self._setup_path_name(path_name)
+        base = {k: y for k, y in self.__dict__.items() if k not in self._forbidden_list}
+        save_json(self.path_name, base, indent=2)
+
+    def to_dict(self):
+        return {k: y for k, y in self.__dict__.items() if k not in self._forbidden_list}
+
+    def set_value(self, var_name: str, value: str) -> None:
+        updateDict(self, {var_name: value})
+
+    def get_value(self, var_name: str) -> Any:
+        return self.__dict__.get(var_name)
+
+    @classmethod
+    def from_dict(
+        cls, dictionary: Dict[str, Any], path: Optional[Union[str, PathLike]] = None
+    ) -> "ModelConfig":
+        assert is_dict(dictionary)
+        return ModelConfig(dictionary, path)
+
+    @classmethod
+    def from_path(cls, path_name: PathLike) -> "ModelConfig":
+        assert is_file(path_name) or is_dir(path_name)
+        settings = {}
+
+        if is_file(path_name):
+            settings.update(load_json(path_name, {}, errors="ignore"))
+        else:
+            files = FileScan.files(
+                path_name,
+                [
+                    "*_config.json",
+                    "config_*.json",
+                    "*_config.json",
+                    "cfg_*.json",
+                    "*_cfg.json",
+                    "cfg.json",
+                    "config.json",
+                    "settings.json",
+                    "settings_*.json",
+                    "*_settings.json",
+                ],
+            )
+            assert files, "No config file found in the provided directory!"
+            settings.update(load_json(files[-1], {}, errors="ignore"))
+        return ModelConfig(settings, path_name)

lt_tensor/datasets/audio.py

@@ -6,11 +6,10 @@ from torch.utils.data import Dataset, DataLoader, Sampler
 from lt_tensor.processors import AudioProcessor
 import torch.nn.functional as FT
 from lt_tensor.misc_utils import log_tensor
+from tqdm import tqdm
 
 
 class WaveMelDataset(Dataset):
-    """Untested!"""
-
     data: Union[list[dict[str, Tensor]], Tuple[Tensor, Tensor]] = []
 
     def __init__(
@@ -19,12 +18,16 @@ class WaveMelDataset(Dataset):
         path: PathLike,
         limit_files: Optional[int] = None,
         max_frame_length: Optional[int] = None,
+        randomize_ranges: bool = False,
     ):
         super().__init__()
         assert max_frame_length is None or max_frame_length >= (
             (audio_processor.n_fft // 2) + 1
         )
+
         self.post_n_fft = (audio_processor.n_fft // 2) + 1
+        if max_frame_length is not None:
+            self.r_range = max(self.post_n_fft + 1, max_frame_length // 3)
         self.ap = audio_processor
         self.files = self.ap.find_audios(path)
         if limit_files:
@@ -32,21 +35,32 @@ class WaveMelDataset(Dataset):
             self.files = self.files[:limit_files]
         self.data = []
 
-        for file in self.files:
-            results = self.load_data(file, max_frame_length)
+        for file in tqdm(self.files, "Loading files"):
+            results = self.load_data(file, max_frame_length, randomize_ranges)
             self.data.extend(results)
 
     def _add_dict(self, audio_raw: Tensor, audio_mel: Tensor, file: PathLike):
         return {"mel": audio_mel, "raw": audio_raw, "file": file}
 
-    def load_data(self, file: PathLike, audio_frames_limit: Optional[int] = None):
-        initial_audio = self.ap.load_audio(file)
+    def load_data(
+        self,
+        file: PathLike,
+        audio_frames_limit: Optional[int] = None,
+        randomize_ranges: bool = False,
+    ):
+        initial_audio = self.ap.rebuild_spectrogram(self.ap.load_audio(file))
         if not audio_frames_limit or initial_audio.shape[-1] <= audio_frames_limit:
+            if initial_audio.shape[-1] < self.post_n_fft:
+                return []
             audio_mel = self.ap.compute_mel(initial_audio, add_base=True)
             return [self._add_dict(initial_audio, audio_mel, file)]
         results = []
+        if randomize_ranges:
+            frame_limit = random.randint(self.r_range, audio_frames_limit)
+        else:
+            frame_limit = audio_frames_limit
         for fragment in torch.split(
-            initial_audio, split_size_or_sections=audio_frames_limit, dim=-1
+            initial_audio, split_size_or_sections=frame_limit, dim=-1
         ):
             if fragment.shape[-1] < self.post_n_fft:
                 # sometimes the tensor will be too small to be able to pass on mel

lt_tensor/losses.py

@@ -1,10 +1,106 @@
-__all__ = ["masked_cross_entropy"]
+__all__ = [
+    "masked_cross_entropy",
+    "adaptive_l1_loss",
+    "contrastive_loss",
+    "smooth_l1_loss",
+    "hybrid_loss",
+    "diff_loss",
+    "cosine_loss",
+    "gan_loss",
+    "ft_n_loss",
+]
 import math
 import random
-from .torch_commons import *
+from lt_tensor.torch_commons import *
 from lt_utils.common import *
 import torch.nn.functional as F
 
+def ft_n_loss(output: Tensor, target: Tensor, weight: Optional[Tensor] = None):
+    if weight is not None:
+        return torch.mean((torch.abs(output - target) + weight) **0.5)
+    return torch.mean(torch.abs(output - target)**0.5)
+
+def adaptive_l1_loss(
+    inp: Tensor,
+    tgt: Tensor,
+    weight: Optional[Tensor] = None,
+    scale: float = 1.0,
+    inverted: bool = False,
+):
+
+    if weight is not None:
+        loss = torch.mean(torch.abs((inp - tgt) + weight.mean()))
+    else:
+        loss = torch.mean(torch.abs(inp - tgt))
+    loss *= scale
+    if inverted:
+        return -loss
+    return loss
+
+
+def smooth_l1_loss(inp: Tensor, tgt: Tensor, beta=1.0, weight=None):
+    diff = torch.abs(inp - tgt)
+    loss = torch.where(diff < beta, 0.5 * diff**2 / beta, diff - 0.5 * beta)
+    if weight is not None:
+        loss *= weight
+    return loss.mean()
+
+
+def contrastive_loss(x1: Tensor, x2: Tensor, label: Tensor, margin: float = 1.0):
+    # label == 1: similar, label == 0: dissimilar
+    dist = torch.nn.functional.pairwise_distance(x1, x2)
+    loss = label * dist**2 + (1 - label) * torch.clamp(margin - dist, min=0.0) ** 2
+    return loss.mean()
+
+
+def cosine_loss(inp, tgt):
+    cos = torch.nn.functional.cosine_similarity(inp, tgt, dim=-1)
+    return 1 - cos.mean()  # Lower is better
+
+
+class GanLosses:
+    @staticmethod
+    def get_loss(
+        pred: Tensor,
+        target_is_real: bool,
+        loss_type: Literal["bce", "mse", "hinge", "wasserstein"] = "bce",
+    ) -> Tensor:
+        if loss_type == "bce":  # Standard GAN
+            target = torch.ones_like(pred) if target_is_real else torch.zeros_like(pred)
+            return F.binary_cross_entropy_with_logits(pred, target)
+
+        elif loss_type == "mse":  # LSGAN
+            target = torch.ones_like(pred) if target_is_real else torch.zeros_like(pred)
+            return F.mse_loss(torch.sigmoid(pred), target)
+
+        elif loss_type == "hinge":
+            if target_is_real:
+                return torch.mean(F.relu(1.0 - pred))
+            else:
+                return torch.mean(F.relu(1.0 + pred))
+
+        elif loss_type == "wasserstein":
+            return -pred.mean() if target_is_real else pred.mean()
+
+        else:
+            raise ValueError(f"Unknown loss_type: {loss_type}")
+
+    @staticmethod
+    def generator_loss(fake_pred: Tensor, loss_type: str = "bce") -> Tensor:
+        return GanLosses.get_loss(fake_pred, target_is_real=True, loss_type=loss_type)
+
+    @staticmethod
+    def discriminator_loss(
+        real_pred: Tensor, fake_pred: Tensor, loss_type: str = "bce"
+    ) -> Tensor:
+        real_loss = GanLosses.get_loss(
+            real_pred, target_is_real=True, loss_type=loss_type
+        )
+        fake_loss = GanLosses.get_loss(
+            fake_pred.detach(), target_is_real=False, loss_type=loss_type
+        )
+        return (real_loss + fake_loss) * 0.5
+
 
 def masked_cross_entropy(
     logits: torch.Tensor,  # [B, T, V]
@@ -61,85 +157,3 @@ def gan_d_loss(real_preds, fake_preds, use_lsgan=True):
                 torch.log(1 - fake + 1e-7)
             )
     return loss
-
-
-def gan_d_loss(real_preds, fake_preds, use_lsgan=True):
-    loss = 0
-    for real, fake in zip(real_preds, fake_preds):
-        if use_lsgan:
-            loss += F.mse_loss(real, torch.ones_like(real)) + F.mse_loss(
-                fake, torch.zeros_like(fake)
-            )
-        else:
-            loss += -torch.mean(torch.log(real + 1e-7)) - torch.mean(
-                torch.log(1 - fake + 1e-7)
-            )
-    return loss
-
-
-def gan_g_loss(fake_preds, use_lsgan=True):
-    loss = 0
-    for fake in fake_preds:
-        if use_lsgan:
-            loss += F.mse_loss(fake, torch.ones_like(fake))
-        else:
-            loss += -torch.mean(torch.log(fake + 1e-7))
-    return loss
-
-
-def feature_matching_loss(real_feats, fake_feats):
-    """real_feats and fake_feats are lists of intermediate features"""
-    loss = 0
-    for real_layers, fake_layers in zip(real_feats, fake_feats):
-        for r, f in zip(real_layers, fake_layers):
-            loss += F.l1_loss(f, r.detach())
-    return loss
-
-
-def feature_loss(real_fmaps, fake_fmaps, weight=2.0):
-    loss = 0.0
-    for dr, dg in zip(real_fmaps, fake_fmaps):  # Each (layer list from a discriminator)
-        for r_feat, g_feat in zip(dr, dg):
-            loss += F.l1_loss(r_feat, g_feat)
-    return loss * weight
-
-
-def discriminator_loss(disc_real_outputs, disc_generated_outputs):
-    loss = 0.0
-    r_losses = []
-    g_losses = []
-
-    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
-        r_loss = F.mse_loss(dr, torch.ones_like(dr))
-        g_loss = F.mse_loss(dg, torch.zeros_like(dg))
-        loss += r_loss + g_loss
-        r_losses.append(r_loss)
-        g_losses.append(g_loss)
-
-    return loss, r_losses, g_losses
-
-
-def generator_loss(fake_outputs):
-    total = 0.0
-    g_losses = []
-    for out in fake_outputs:
-        loss = F.mse_loss(out, torch.ones_like(out))
-        g_losses.append(loss)
-        total += loss
-    return total, g_losses
-
-
-def multi_resolution_stft_loss(y, y_hat, fft_sizes=[512, 1024, 2048]):
-    loss = 0
-    for fft_size in fft_sizes:
-        hop = fft_size // 4
-        win = fft_size
-        y_stft = torch.stft(
-            y, n_fft=fft_size, hop_length=hop, win_length=win, return_complex=True
-        )
-        y_hat_stft = torch.stft(
-            y_hat, n_fft=fft_size, hop_length=hop, win_length=win, return_complex=True
-        )
-
-        loss += F.l1_loss(torch.abs(y_stft), torch.abs(y_hat_stft))
-    return loss

lt_tensor/math_ops.py

@@ -12,7 +12,7 @@ __all__ = [
     "phase",
 ]
 
-from .torch_commons import *
+from lt_tensor.torch_commons import *
 
 
 def sin_tensor(x: Tensor, freq: float = 1.0) -> Tensor:

lt_tensor/misc_utils.py

@@ -15,11 +15,14 @@ __all__ = [
     "TorchCacheUtils",
     "clear_cache",
     "default_device",
+    "soft_restore",
     "Packing",
     "Padding",
     "Masking",
     "LogTensor",
     "get_losses",
+    "plot_view",
+    "get_weights",
 ]
 
 import re
@@ -27,22 +30,104 @@ import gc
 import sys
 import random
 import numpy as np
-from lt_utils.type_utils import is_str
-from .torch_commons import *
+import warnings
+from lt_utils.type_utils import is_str, is_dir, is_file, is_pathlike, is_path_valid
+from lt_utils.file_ops import FileScan, find_files, path_to_str, load_json, load_yaml
+from lt_tensor.torch_commons import *
 from lt_utils.misc_utils import cache_wrapper
 from lt_utils.common import *
 from lt_utils.misc_utils import ff_list
 import torch.nn.functional as F
 
 
+def plot_view(
+    data: Dict[str, List[Any]],
+    title: str = "Loss",
+    max_amount: int = 0,
+    xaxis_title="Step/Epoch",
+    yaxis_title="Loss",
+    template="plotly_dark",
+):
+    try:
+        import plotly.graph_objs as go
+    except ModuleNotFoundError:
+        warnings.warn(
+            "No installation of plotly was found. To use it use 'pip install plotly' and restart this application!"
+        )
+        return
+    fig = go.Figure()
+    for mode, values in data.items():
+        if values:
+            items = values if not max_amount > 0 else values[-max_amount:]
+            fig.add_trace(go.Scatter(y=items, name=mode.capitalize()))
+    fig.update_layout(
+        title=title,
+        xaxis_title=xaxis_title,
+        yaxis_title=yaxis_title,
+        template=template,
+    )
+    return fig
+
+
+def get_weights(directory: Union[str, PathLike]):
+    is_path_valid(directory, validate=True)  # raises validation if its invalid path
+    directory = Path(directory)
+    if is_file(directory):
+        if directory.name.endswith((".pt", ".ckpt", ".pth")):
+            return directory
+        directory = directory.parent
+    res = sorted(find_files(directory, ["*.pt", "*.ckpt", "*.pth"]))
+    return res[-1] if res else None
+
+
+def get_config(directory: Union[str, PathLike], default: Optional[Any] = None):
+    # raises validation if its invalid path only when default is None otherwise it returns the defaults.
+    if not is_path_valid(directory, validate=default is None):
+        return default
+    directory = Path(directory)
+    if is_file(directory):
+        if directory.name.endswith((".json", ".yaml", ".yml")):
+            if directory.name.endswith(".json"):
+                return load_json(directory, default)
+            return load_yaml(directory, default)
+        directory = directory.parent
+    res = sorted(find_files(directory, ["*.pt", "*.ckpt", "*.pth"]))
+    if res:
+        res = res[-1]
+        if Path(res).name.endswith(".json"):
+            return load_json(directory, default)
+        return load_yaml(directory, default)
+    return default
+
+
+def updateDict(self, dct: dict[str, Any]):
+    for k, v in dct.items():
+        setattr(self, k, v)
+
+
+def soft_restore(tensor, epsilon=1e-6):
+    return torch.where(tensor == 0, torch.full_like(tensor, epsilon), tensor)
+
+
 def try_torch(fn: str, *args, **kwargs):
+    tryed_torch = False
+    not_present_message = (
+        f"Both `torch` and `torch.nn.functional` does not contain the module `{fn}`"
+    )
     try:
-        return getattr(F, fn)(*args, **kwargs)
-    except Exception as e:
-        try:
+        if hasattr(F, fn):
+            return getattr(F, fn)(*args, **kwargs)
+        elif hasattr(torch, fn):
+            tryed_torch = True
             return getattr(torch, fn)(*args, **kwargs)
+        return not_present_message
+    except Exception as a:
+        try:
+            if not tryed_torch and hasattr(torch, fn):
+                return getattr(torch, fn)(*args, **kwargs)
+            return str(a)
         except Exception as e:
-            return str(e)
+            return str(e) + " | " + str(a)
 
 
 def log_tensor(
@@ -192,6 +277,7 @@ class LogTensor:
             ("max", dict(dim=-1)),
         ],
         validate_item_type: bool = False,
+        exclude_invalid_losses: bool = True,
         **kwargs,
     ):
         invalid_type = not isinstance(inputs, (Tensor, np.ndarray, list, tuple))
@@ -243,12 +329,13 @@ class LogTensor:
                     value = try_torch(log_fn, inputs)
                 else:
                     value = try_torch(log_fn, inputs, log_args)
+
                 results = self._process(log_fn, value)
                 current_register[log_fn] = results
             self.do_print = old_print
 
         if target is not None:
-            losses = get_losses(inputs, target, False)
+            losses = get_losses(inputs, target, exclude_invalid_losses)
             started_ls = False
             if self.do_print:
                 for loss, res in losses.items():