lt-tensor 0.0.1a13__py3-none-any.whl → 0.0.1a15__py3-none-any.whl

lt_tensor/model_zoo/istft/generator.py

@@ -1,52 +1,7 @@
-__all__ = ["iSTFTGenerator", "ResBlocks"]
-import gc
-import math
-import itertools
+__all__ = ["iSTFTGenerator"]
 from lt_utils.common import *
 from lt_tensor.torch_commons import *
-from lt_tensor.model_base import Model
-from lt_tensor.misc_utils import log_tensor
-from lt_tensor.model_zoo.residual import ResBlock1D, ConvNets, get_weight_norm
-from lt_utils.misc_utils import log_traceback
-from lt_tensor.processors import AudioProcessor
-from lt_utils.type_utils import is_dir, is_pathlike
-from lt_tensor.misc_utils import set_seed, clear_cache
-from lt_tensor.model_zoo.discriminator import MultiPeriodDiscriminator, MultiScaleDiscriminator
-import torch.nn.functional as F
-from lt_tensor.config_templates import updateDict, ModelConfig
-
-
-class ResBlocks(ConvNets):
-    def __init__(
-        self,
-        channels: int,
-        resblock_kernel_sizes: List[Union[int, List[int]]] = [3, 7, 11],
-        resblock_dilation_sizes: List[Union[int, List[int]]] = [
-            [1, 3, 5],
-            [1, 3, 5],
-            [1, 3, 5],
-        ],
-        activation: nn.Module = nn.LeakyReLU(0.1),
-    ):
-        super().__init__()
-        self.num_kernels = len(resblock_kernel_sizes)
-        self.rb = nn.ModuleList()
-        self.activation = activation
-
-        for k, j in zip(resblock_kernel_sizes, resblock_dilation_sizes):
-            self.rb.append(ResBlock1D(channels, k, j, activation))
-
-        self.rb.apply(self.init_weights)
-
-    def forward(self, x: torch.Tensor):
-        xs = None
-        for i, block in enumerate(self.rb):
-            if i == 0:
-                xs = block(x)
-            else:
-                xs += block(x)
-        x = xs / self.num_kernels
-        return self.activation(x)
+from lt_tensor.model_zoo.residual import ConvNets, ResBlocks1D, ResBlock1D, ResBlock1D2
 
 
 class iSTFTGenerator(ConvNets):
@@ -65,6 +20,7 @@ class iSTFTGenerator(ConvNets):
         n_fft: int = 16,
         activation: nn.Module = nn.LeakyReLU(0.1),
         hop_length: int = 256,
+        residual_cls: Union[ResBlock1D, ResBlock1D2] = ResBlock1D
     ):
         super().__init__()
         self.num_kernels = len(resblock_kernel_sizes)
@@ -82,6 +38,7 @@ class iSTFTGenerator(ConvNets):
                     upsample_initial_channel,
                     resblock_kernel_sizes,
                     resblock_dilation_sizes,
+                    residual_cls
                 )
             )
 
@@ -91,25 +48,13 @@ class iSTFTGenerator(ConvNets):
         self.conv_post.apply(self.init_weights)
         self.reflection_pad = nn.ReflectionPad1d((1, 0))
 
-        self.phase = nn.Sequential(
-            nn.LeakyReLU(0.2),
-            nn.Conv1d(self.post_n_fft, self.post_n_fft, kernel_size=3, padding=1),
-            nn.LeakyReLU(0.2),
-            nn.Conv1d(self.post_n_fft, self.post_n_fft, kernel_size=3, padding=1),
-        )
-        self.spec = nn.Sequential(
-            nn.LeakyReLU(0.2),
-            nn.Conv1d(self.post_n_fft, self.post_n_fft, kernel_size=3, padding=1),
-            nn.LeakyReLU(0.2),
-            nn.Conv1d(self.post_n_fft, self.post_n_fft, kernel_size=3, padding=1),
-        )
-
     def _make_blocks(
         self,
         state: Tuple[int, int, int],
         upsample_initial_channel: int,
         resblock_kernel_sizes: List[Union[int, List[int]]],
         resblock_dilation_sizes: List[int | List[int]],
+        residual: nn.Module
     ):
         i, k, u = state
         channels = upsample_initial_channel // (2 ** (i + 1))
@@ -127,11 +72,12 @@ class iSTFTGenerator(ConvNets):
                         )
                     ).apply(self.init_weights),
                 ),
-                residual=ResBlocks(
+                residual=ResBlocks1D(
                     channels,
                     resblock_kernel_sizes,
                     resblock_dilation_sizes,
                     self.activation,
+                    residual
                 ),
             )
         )
@@ -142,9 +88,7 @@ class iSTFTGenerator(ConvNets):
             x = block["up"](x)
             x = block["residual"](x)
 
-        x = self.reflection_pad(x)
-        x = self.conv_post(x)
-        spec = torch.exp(self.spec(x[:, : self.post_n_fft, :]))
-        phase = torch.sin(self.phase(x[:, self.post_n_fft :, :]))
-
+        x = self.conv_post(self.activation(self.reflection_pad(x)))
+        spec = torch.exp(x[:, : self.post_n_fft, :])
+        phase = torch.sin(x[:, self.post_n_fft :, :])
         return spec, phase

lt_tensor/model_zoo/istft/trainer.py

@@ -1,20 +1,21 @@
-__all__ = ["AudioSettings", "AudioDecoder"]
+__all__ = ["AudioSettings", "AudioDecoderTrainer", "AudioGeneratorOnlyTrainer"]
 import gc
-import math
 import itertools
 from lt_utils.common import *
 import torch.nn.functional as F
 from lt_tensor.torch_commons import *
 from lt_tensor.model_base import Model
-from lt_tensor.misc_utils import log_tensor
 from lt_utils.misc_utils import log_traceback
 from lt_tensor.processors import AudioProcessor
 from lt_tensor.misc_utils import set_seed, clear_cache
-from lt_utils.type_utils import is_dir, is_pathlike, is_file
-from lt_tensor.config_templates import updateDict, ModelConfig
+from lt_utils.type_utils import is_dir, is_pathlike
+from lt_tensor.config_templates import ModelConfig
 from lt_tensor.model_zoo.istft.generator import iSTFTGenerator
-from lt_tensor.model_zoo.residual import ResBlock1D, ConvNets, get_weight_norm
-from lt_tensor.model_zoo.discriminator import MultiPeriodDiscriminator, MultiScaleDiscriminator
+from lt_tensor.model_zoo.discriminator import (
+    MultiPeriodDiscriminator,
+    MultiScaleDiscriminator,
+)
+from lt_tensor.model_zoo.residual import ResBlock1D2, ResBlock1D
 
 
 def feature_loss(fmap_r, fmap_g):
@@ -29,7 +30,6 @@ def generator_adv_loss(disc_outputs):
     loss = 0
     for dg in disc_outputs:
         l = torch.mean((1 - dg) ** 2)
-
         loss += l
     return loss
 
@@ -44,29 +44,6 @@ def discriminator_loss(disc_real_outputs, disc_generated_outputs):
     return loss
 
 
-"""def feature_loss(fmap_r, fmap_g):
-    loss = 0
-    for dr, dg in zip(fmap_r, fmap_g):
-        for rl, gl in zip(dr, dg):
-            loss += torch.mean(torch.abs(rl - gl))
-    return loss * 2
-
-
-def generator_adv_loss(fake_preds):
-    loss = 0.0
-    for f in fake_preds:
-        loss += torch.mean((f - 1.0) ** 2)
-    return loss
-
-
-def discriminator_loss(real_preds, fake_preds):
-    loss = 0.0
-    for r, f in zip(real_preds, fake_preds):
-        loss += torch.mean((r - 1.0) ** 2) + torch.mean(f**2)
-    return loss
-"""
-
-
 class AudioSettings(ModelConfig):
     def __init__(
         self,
@@ -90,6 +67,7 @@ class AudioSettings(ModelConfig):
         scheduler_template: Callable[
             [optim.Optimizer], optim.lr_scheduler.LRScheduler
         ] = lambda optimizer: optim.lr_scheduler.ExponentialLR(optimizer, gamma=0.998),
+        residual_cls: Union[ResBlock1D, ResBlock1D2] = ResBlock1D,
     ):
         self.in_channels = n_mels
         self.upsample_rates = upsample_rates
@@ -105,14 +83,15 @@ class AudioSettings(ModelConfig):
         self.lr = lr
         self.adamw_betas = adamw_betas
         self.scheduler_template = scheduler_template
+        self.residual_cls = residual_cls
 
 
-class AudioDecoder(Model):
+class AudioDecoderTrainer(Model):
     def __init__(
         self,
         audio_processor: AudioProcessor,
         settings: Optional[AudioSettings] = None,
-        generator: Optional[Union[Model, "iSTFTGenerator"]] = None,  # non initalized!
+        generator: Optional[Union[Model, "iSTFTGenerator"]] = None,  # non initialized!
     ):
         super().__init__()
         if settings is None:
@@ -175,14 +154,20 @@ class AudioDecoder(Model):
         return True
 
     def update_schedulers_and_optimizer(self):
+        gc.collect()
+        self.g_optim = None
+        self.g_scheduler = None
+        gc.collect()
         self.g_optim = optim.AdamW(
             self.generator.parameters(),
             lr=self.settings.lr,
             betas=self.settings.adamw_betas,
         )
+        gc.collect()
         self.g_scheduler = self.settings.scheduler_template(self.g_optim)
         if any([self.mpd is None, self.msd is None]):
             return
+        gc.collect()
         self.d_optim = optim.AdamW(
             itertools.chain(self.mpd.parameters(), self.msd.parameters()),
             lr=self.settings.lr,
@@ -274,9 +259,9 @@ class AudioDecoder(Model):
             win_length=self.settings.n_fft,
         )
         if not return_dict:
-            return wave[:, : wave.shape[-1] - 256]
+            return wave
         return {
-            "wave": wave[:, : wave.shape[-1] - 256],
+            "wave": wave,
             "spec": spec,
             "phase": phase,
         }
@@ -310,8 +295,8 @@ class AudioDecoder(Model):
             self.settings.n_fft,
             hop_length=4,
             win_length=self.settings.n_fft,
-            # length=real_audio.shape[-1]
-        )[:, : real_audio.shape[-1]]
+            length=real_audio.shape[-1],
+        )
 
         disc_kwargs = dict(
             real_audio=real_audio,
@@ -324,7 +309,7 @@ class AudioDecoder(Model):
         else:
             disc_out = self._discriminator_step(**disc_kwargs)
 
-        generato_kwargs = dict(
+        generator_kwargs = dict(
             mels=mels,
             real_audio=real_audio,
             fake_audio=fake_audio,
@@ -339,8 +324,8 @@ class AudioDecoder(Model):
 
         if is_generator_frozen:
             with torch.no_grad():
-                return self._generator_step(**generato_kwargs)
-        return self._generator_step(**generato_kwargs)
+                return self._generator_step(**generator_kwargs)
+        return self._generator_step(**generator_kwargs)
 
     def _discriminator_step(
         self,
@@ -349,7 +334,8 @@ class AudioDecoder(Model):
         am_i_frozen: bool = False,
     ):
         # ========== Discriminator Forward Pass ==========
-
+        if not am_i_frozen:
+            self.d_optim.zero_grad()
         # MPD
         real_mpd_preds, _ = self.mpd(real_audio)
         fake_mpd_preds, _ = self.mpd(fake_audio)
@@ -362,7 +348,6 @@ class AudioDecoder(Model):
         loss_d = loss_d_mpd + loss_d_msd
 
         if not am_i_frozen:
-            self.d_optim.zero_grad()
             loss_d.backward()
             self.d_optim.step()
 
@@ -384,6 +369,8 @@ class AudioDecoder(Model):
         am_i_frozen: bool = False,
     ):
         # ========== Generator Loss ==========
+        if not am_i_frozen:
+            self.g_optim.zero_grad()
         real_mpd_feats = self.mpd(real_audio)[1]
         real_msd_feats = self.msd(real_audio)[1]
 
@@ -395,7 +382,7 @@ class AudioDecoder(Model):
         loss_fm_mpd = feature_loss(real_mpd_feats, fake_mpd_feats)
         loss_fm_msd = feature_loss(real_msd_feats, fake_msd_feats)
 
-        loss_stft = self.audio_processor.stft_loss(fake_audio, real_audio) * stft_scale
+        # loss_stft = self.audio_processor.stft_loss(fake_audio, real_audio) * stft_scale
         loss_mel = (
             F.huber_loss(self.audio_processor.compute_mel(fake_audio), mels) * mel_scale
         )
@@ -403,20 +390,21 @@ class AudioDecoder(Model):
 
         loss_adv = (loss_adv_mpd + loss_adv_msd) * adv_scale
 
-        loss_g = loss_adv + loss_fm + loss_stft  # + loss_mel
+        loss_g = loss_adv + loss_fm + loss_mel  # + loss_stft
         if not am_i_frozen:
-            self.g_optim.zero_grad()
             loss_g.backward()
             self.g_optim.step()
+
+        lr_g, lr_d = self.get_lr()
         return {
             "loss_g": loss_g.item(),
             "loss_d": loss_d,
             "loss_adv": loss_adv.item(),
             "loss_fm": loss_fm.item(),
-            "loss_stft": loss_stft.item(),
+            "loss_stft": 1.0,  # loss_stft.item(),
             "loss_mel": loss_mel.item(),
-            "lr_g": self.g_optim.param_groups[0]["lr"],
-            "lr_d": self.d_optim.param_groups[0]["lr"],
+            "lr_g": lr_g,
+            "lr_d": lr_d,
         }
 
     def step_scheduler(
@@ -442,34 +430,198 @@ class AudioDecoder(Model):
             self.g_scheduler = self.settings.scheduler_template(self.g_optim)
 
 
-class ResBlocks(ConvNets):
+class AudioGeneratorOnlyTrainer(Model):
     def __init__(
         self,
-        channels: int,
-        resblock_kernel_sizes: List[Union[int, List[int]]] = [3, 7, 11],
-        resblock_dilation_sizes: List[Union[int, List[int]]] = [
-            [1, 3, 5],
-            [1, 3, 5],
-            [1, 3, 5],
-        ],
-        activation: nn.Module = nn.LeakyReLU(0.1),
+        audio_processor: AudioProcessor,
+        settings: Optional[AudioSettings] = None,
+        generator: Optional[Union[Model, "iSTFTGenerator"]] = None,  # non initialized!
     ):
         super().__init__()
-        self.num_kernels = len(resblock_kernel_sizes)
-        self.rb = nn.ModuleList()
-        self.activation = activation
+        if settings is None:
+            self.settings = AudioSettings()
+        elif isinstance(settings, dict):
+            self.settings = AudioSettings(**settings)
+        elif isinstance(settings, AudioSettings):
+            self.settings = settings
+        else:
+            raise ValueError(
+                "Cannot initialize the waveDecoder with the given settings. "
+                "Use either a dictionary, or the class WaveSettings to setup the settings. "
+                "Alternatively, leave it None to use the default values."
+            )
+        if self.settings.seed is not None:
+            set_seed(self.settings.seed)
+        if generator is None:
+            generator = iSTFTGenerator
+        self.generator: iSTFTGenerator = generator(
+            in_channels=self.settings.in_channels,
+            upsample_rates=self.settings.upsample_rates,
+            upsample_kernel_sizes=self.settings.upsample_kernel_sizes,
+            upsample_initial_channel=self.settings.upsample_initial_channel,
+            resblock_kernel_sizes=self.settings.resblock_kernel_sizes,
+            resblock_dilation_sizes=self.settings.resblock_dilation_sizes,
+            n_fft=self.settings.n_fft,
+            activation=self.settings.activation,
+        )
+        self.generator.eval()
+        self.gen_training = False
+        self.audio_processor = audio_processor
+
+    def setup_training_mode(self, *args, **kwargs):
+        self.finish_training_setup()
+        self.update_schedulers_and_optimizer()
+        self.gen_training = True
+        return True
+
+    def update_schedulers_and_optimizer(self):
+        self.g_optim = optim.AdamW(
+            self.generator.parameters(),
+            lr=self.settings.lr,
+            betas=self.settings.adamw_betas,
+        )
+        self.g_scheduler = self.settings.scheduler_template(self.g_optim)
+
+    def set_lr(self, new_lr: float = 1e-4):
+        if self.g_optim is not None:
+            for groups in self.g_optim.param_groups:
+                groups["lr"] = new_lr
+        return self.get_lr()
+
+    def get_lr(self) -> Tuple[float, float]:
+        if self.g_optim is not None:
+            return self.g_optim.param_groups[0]["lr"]
+        return float("nan")
+
+    def save_weights(self, path, replace=True):
+        is_pathlike(path, check_if_empty=True, validate=True)
+        if str(path).endswith(".pt"):
+            path = Path(path).parent
+        else:
+            path = Path(path)
+        self.generator.save_weights(Path(path, "generator.pt"), replace)
 
-        for k, j in zip(resblock_kernel_sizes, resblock_dilation_sizes):
-            self.rb.append(ResBlock1D(channels, k, j, activation))
+    def load_weights(
+        self,
+        path,
+        raise_if_not_exists=False,
+        strict=True,
+        assign=False,
+        weights_only=False,
+        mmap=None,
+        **torch_loader_kwargs
+    ):
+        is_pathlike(path, check_if_empty=True, validate=True)
+        if str(path).endswith(".pt"):
+            path = Path(path)
+        else:
+            path = Path(path, "generator.pt")
 
-        self.rb.apply(self.init_weights)
+        self.generator.load_weights(
+            path,
+            raise_if_not_exists,
+            strict,
+            assign,
+            weights_only,
+            mmap,
+            **torch_loader_kwargs,
+        )
+
+    def finish_training_setup(self):
+        gc.collect()
+        clear_cache()
+        self.eval()
+        self.gen_training = False
+
+    def forward(self, mel_spec: Tensor) -> Tuple[Tensor, Tensor]:
+        """Returns the generated spec and phase"""
+        return self.generator.forward(mel_spec)
 
-    def forward(self, x: torch.Tensor):
-        xs = None
-        for i, block in enumerate(self.rb):
-            if i == 0:
-                xs = block(x)
-            else:
-                xs += block(x)
-        x = xs / self.num_kernels
-        return self.activation(x)
+    def inference(
+        self,
+        mel_spec: Tensor,
+        return_dict: bool = False,
+    ) -> Union[Dict[str, Tensor], Tensor]:
+        spec, phase = super().inference(mel_spec)
+        wave = self.audio_processor.inverse_transform(
+            spec,
+            phase,
+            self.settings.n_fft,
+            hop_length=4,
+            win_length=self.settings.n_fft,
+        )
+        if not return_dict:
+            return wave[:, : wave.shape[-1] - 256]
+        return {
+            "wave": wave[:, : wave.shape[-1] - 256],
+            "spec": spec,
+            "phase": phase,
+        }
+
+    def set_device(self, device: str):
+        self.to(device=device)
+        self.generator.to(device=device)
+        self.audio_processor.to(device=device)
+        self.msd.to(device=device)
+        self.mpd.to(device=device)
+
+    def train_step(
+        self,
+        mels: Tensor,
+        real_audio: Tensor,
+        stft_scale: float = 1.0,
+        mel_scale: float = 1.0,
+        ext_loss: Optional[Callable[[Tensor, Tensor], Tensor]] = None,
+    ):
+        if not self.gen_training:
+            self.setup_training_mode()
+
+        self.g_optim.zero_grad()
+        spec, phase = self.generator.train_step(mels)
+
+        real_audio = real_audio.squeeze(1)
+        with torch.no_grad():
+            fake_audio = self.audio_processor.inverse_transform(
+                spec,
+                phase,
+                self.settings.n_fft,
+                hop_length=4,
+                win_length=self.settings.n_fft,
+            )[:, : real_audio.shape[-1]]
+        loss_stft = self.audio_processor.stft_loss(fake_audio, real_audio) * stft_scale
+        loss_mel = (
+            F.huber_loss(self.audio_processor.compute_mel(fake_audio), mels) * mel_scale
+        )
+        loss_g.backward()
+        loss_g = loss_stft + loss_mel
+        loss_ext = 0
+
+        if ext_loss is not None:
+            l_ext = ext_loss(fake_audio, real_audio)
+            loss_g = loss_g + l_ext
+            loss_ext = l_ext.item()
+
+        self.g_optim.step()
+        return {
+            "loss": loss_g.item(),
+            "loss_stft": loss_stft.item(),
+            "loss_mel": loss_mel.item(),
+            "loss_ext": loss_ext,
+            "lr": self.get_lr(),
+        }
+
+    def step_scheduler(self):
+
+        if self.g_scheduler is not None:
+            self.g_scheduler.step()
+
+    def reset_schedulers(self, lr: Optional[float] = None):
+        """
+        In case you have adopted another strategy, with this function,
+        it is possible restart the scheduler and set the lr to another value.
+        """
+        if lr is not None:
+            self.set_lr(lr)
+        if self.g_optim is not None:
+            self.g_scheduler = None
+            self.g_scheduler = self.settings.scheduler_template(self.g_optim)