lt-tensor 0.0.1a12__py3-none-any.whl → 0.0.1a14__py3-none-any.whl

lt_tensor/model_zoo/istft/sg.py

@@ -0,0 +1,142 @@
+import torch
+import torch.nn as nn
+import math
+from einops import repeat
+
+
+class SineGen(nn.Module):
+    def __init__(
+        self,
+        samp_rate,
+        upsample_scale,
+        harmonic_num=0,
+        sine_amp=0.1,
+        noise_std=0.003,
+        voiced_threshold=0,
+        flag_for_pulse=False,
+    ):
+        super().__init__()
+        self.sampling_rate = samp_rate
+        self.upsample_scale = upsample_scale
+        self.harmonic_num = harmonic_num
+        self.sine_amp = sine_amp
+        self.noise_std = noise_std
+        self.voiced_threshold = voiced_threshold
+        self.flag_for_pulse = flag_for_pulse
+        self.dim = self.harmonic_num + 1  # fundamental + harmonics
+
+    def _f02uv_b(self, f0):
+        return (f0 > self.voiced_threshold).float()  # [B, T]
+
+    def _f02uv(self, f0):
+        return (f0 > self.voiced_threshold).float().unsqueeze(-1)  # -> (B, T, 1)
+
+    @torch.no_grad()
+    def _f02sine(self, f0_values):
+        """
+        f0_values: (B, T, 1)
+        Output: sine waves (B, T * upsample, dim)
+        """
+        B, T, _ = f0_values.size()
+        f0_upsampled = repeat(
+            f0_values, "b t d -> b (t r) d", r=self.upsample_scale
+        )  # (B, T_up, 1)
+
+        # Create harmonics
+        harmonics = (
+            torch.arange(1, self.dim + 1, device=f0_values.device)
+            .float()
+            .view(1, 1, -1)
+        )
+        f0_harm = f0_upsampled * harmonics  # (B, T_up, dim)
+
+        # Convert Hz to radians (2πf/sr), then integrate to get phase
+        rad_values = f0_harm / self.sampling_rate  # normalized freq
+        rad_values = rad_values % 1.0  # remove multiples of 2π
+
+        # Random initial phase for each harmonic (except 0th if pulse mode)
+        if self.flag_for_pulse:
+            rand_ini = torch.zeros((B, 1, self.dim), device=f0_values.device)
+        else:
+            rand_ini = torch.rand((B, 1, self.dim), device=f0_values.device)
+
+        rand_ini = rand_ini * 2 * math.pi
+
+        # Compute cumulative phase
+        rad_values = rad_values * 2 * math.pi
+        phase = torch.cumsum(rad_values, dim=1) + rand_ini  # (B, T_up, dim)
+
+        sine_waves = torch.sin(phase)  # (B, T_up, dim)
+        return sine_waves
+
+    def _forward(self, f0):
+        """
+        f0: (B, T, 1)
+        returns: sine signal with harmonics and noise added
+        """
+        sine_waves = self._f02sine(f0)  # (B, T_up, dim)
+        uv = self._f02uv_b(f0) # (B, T, 1)
+        uv = repeat(uv, "b t d -> b (t r) d", r=self.upsample_scale)  # (B, T_up, 1)
+
+        # voiced sine + unvoiced noise
+        sine_signal = self.sine_amp * sine_waves * uv  # (B, T_up, dim)
+        noise = torch.randn_like(sine_signal) * self.noise_std
+        output = sine_signal + noise * (1.0 - uv)  # noise added only on unvoiced
+
+        return output  # (B, T_up, dim)
+
+    def forward(self, f0):
+        """
+        Args:
+            f0: (B, T) in Hz (before upsampling)
+        Returns:
+            sine_waves: (B, T_up, dim)
+            uv: (B, T_up, 1)
+            noise: (B, T_up, 1)
+        """
+        B, T = f0.shape
+        device = f0.device
+
+        # Get uv mask (before upsampling)
+        uv = self._f02uv(f0)  # (B, T, 1)
+
+        # Expand f0 to include harmonics: (B, T, dim)
+        f0 = f0.unsqueeze(-1)  # (B, T, 1)
+        harmonics = (
+            torch.arange(1, self.dim + 1, device=device).float().view(1, 1, -1)
+        )  # (1, 1, dim)
+        f0_harm = f0 * harmonics  # (B, T, dim)
+
+        # Upsample
+        f0_harm_up = repeat(
+            f0_harm, "b t d -> b (t r) d", r=self.upsample_scale
+        )  # (B, T_up, dim)
+        uv_up = repeat(uv, "b t d -> b (t r) d", r=self.upsample_scale)  # (B, T_up, 1)
+
+        # Convert to radians
+        rad_per_sample = f0_harm_up / self.sampling_rate  # Hz → cycles/sample
+        rad_per_sample = rad_per_sample * 2 * math.pi  # cycles → radians/sample
+
+        # Random phase init for each sample
+        B, T_up, D = rad_per_sample.shape
+        rand_phase = torch.rand(B, D, device=device) * 2 * math.pi  # (B, D)
+
+        # Compute cumulative phase
+        phase = torch.cumsum(rad_per_sample, dim=1) + rand_phase.unsqueeze(
+            1
+        )  # (B, T_up, D)
+
+        # Apply sine
+        sine_waves = torch.sin(phase) * self.sine_amp  # (B, T_up, D)
+
+        # Handle unvoiced: create noise only for fundamental
+        noise = torch.randn(B, T_up, 1, device=device) * self.noise_std
+        if self.flag_for_pulse:
+            # If pulse mode is on, align phase at start of voiced segments
+            # Optional and tricky to implement — may require segmenting uv
+            pass
+
+        # Replace sine by noise for unvoiced (only on fundamental)
+        sine_waves[:, :, 0:1] = sine_waves[:, :, 0:1] * uv_up + noise * (1 - uv_up)
+
+        return sine_waves, uv_up, noise

lt_tensor/model_zoo/istft/trainer.py

@@ -1,41 +1,45 @@
-__all__ = ["AudioSettings", "AudioDecoder"]
+__all__ = ["AudioSettings", "AudioDecoderTrainer", "AudioGeneratorOnlyTrainer"]
 import gc
-import math
 import itertools
 from lt_utils.common import *
 import torch.nn.functional as F
 from lt_tensor.torch_commons import *
 from lt_tensor.model_base import Model
-from lt_tensor.misc_utils import log_tensor
 from lt_utils.misc_utils import log_traceback
 from lt_tensor.processors import AudioProcessor
 from lt_tensor.misc_utils import set_seed, clear_cache
-from lt_utils.type_utils import is_dir, is_pathlike, is_file
-from lt_tensor.config_templates import updateDict, ModelConfig
+from lt_utils.type_utils import is_dir, is_pathlike
+from lt_tensor.config_templates import ModelConfig
 from lt_tensor.model_zoo.istft.generator import iSTFTGenerator
-from lt_tensor.model_zoo.rsd import ResBlock1D, ConvNets, get_weight_norm
-from lt_tensor.model_zoo.disc import MultiPeriodDiscriminator, MultiScaleDiscriminator
+from lt_tensor.model_zoo.discriminator import (
+    MultiPeriodDiscriminator,
+    MultiScaleDiscriminator,
+)
 
 
-def feature_loss(real_feats, fake_feats):
-    loss = 0.0
-    for r, f in zip(real_feats, fake_feats):
-        for ri, fi in zip(r, f):
-            loss += F.l1_loss(ri, fi)
-    return loss
+def feature_loss(fmap_r, fmap_g):
+    loss = 0
+    for dr, dg in zip(fmap_r, fmap_g):
+        for rl, gl in zip(dr, dg):
+            loss += torch.mean(torch.abs(rl - gl))
+    return loss * 2
 
 
-def generator_adv_loss(fake_preds):
-    loss = 0.0
-    for f in fake_preds:
-        loss += torch.mean((f - 1.0) ** 2)
+def generator_adv_loss(disc_outputs):
+    loss = 0
+    for dg in disc_outputs:
+        l = torch.mean((1 - dg) ** 2)
+        loss += l
     return loss
 
 
-def discriminator_loss(real_preds, fake_preds):
-    loss = 0.0
-    for r, f in zip(real_preds, fake_preds):
-        loss += torch.mean((r - 1.0) ** 2) + torch.mean(f**2)
+def discriminator_loss(disc_real_outputs, disc_generated_outputs):
+    loss = 0
+
+    for dr, dg in zip(disc_real_outputs, disc_generated_outputs):
+        r_loss = torch.mean((1 - dr) ** 2)
+        g_loss = torch.mean(dg**2)
+        loss += r_loss + g_loss
     return loss
 
 
@@ -79,12 +83,12 @@ class AudioSettings(ModelConfig):
         self.scheduler_template = scheduler_template
 
 
-class AudioDecoder(Model):
+class AudioDecoderTrainer(Model):
     def __init__(
         self,
         audio_processor: AudioProcessor,
         settings: Optional[AudioSettings] = None,
-        generator: Optional[Union[Model, "iSTFTGenerator"]] = None,  # non initalized!
+        generator: Optional[Union[Model, "iSTFTGenerator"]] = None,  # non initialized!
     ):
         super().__init__()
         if settings is None:
@@ -284,9 +288,6 @@ class AudioDecoder(Model):
             win_length=self.settings.n_fft,
             # length=real_audio.shape[-1]
         )[:, : real_audio.shape[-1]]
-        # smallest = min(real_audio.shape[-1], fake_audio.shape[-1])
-        # real_audio = real_audio[:, :, :smallest].squeeze(1)
-        # fake_audio = fake_audio[:, :smallest]
 
         disc_kwargs = dict(
             real_audio=real_audio,
@@ -299,7 +300,7 @@ class AudioDecoder(Model):
         else:
             disc_out = self._discriminator_step(**disc_kwargs)
 
-        generato_kwargs = dict(
+        generator_kwargs = dict(
             mels=mels,
             real_audio=real_audio,
             fake_audio=fake_audio,
@@ -314,8 +315,8 @@ class AudioDecoder(Model):
 
         if is_generator_frozen:
             with torch.no_grad():
-                return self._generator_step(**generato_kwargs)
-        return self._generator_step(**generato_kwargs)
+                return self._generator_step(**generator_kwargs)
+        return self._generator_step(**generator_kwargs)
 
     def _discriminator_step(
         self,
@@ -324,7 +325,8 @@ class AudioDecoder(Model):
         am_i_frozen: bool = False,
     ):
         # ========== Discriminator Forward Pass ==========
-
+        if not am_i_frozen:
+            self.d_optim.zero_grad()
         # MPD
         real_mpd_preds, _ = self.mpd(real_audio)
         fake_mpd_preds, _ = self.mpd(fake_audio)
@@ -337,7 +339,6 @@ class AudioDecoder(Model):
         loss_d = loss_d_mpd + loss_d_msd
 
         if not am_i_frozen:
-            self.d_optim.zero_grad()
             loss_d.backward()
             self.d_optim.step()
 
@@ -359,6 +360,8 @@ class AudioDecoder(Model):
         am_i_frozen: bool = False,
     ):
         # ========== Generator Loss ==========
+        if not am_i_frozen:
+            self.g_optim.zero_grad()
         real_mpd_feats = self.mpd(real_audio)[1]
         real_msd_feats = self.msd(real_audio)[1]
 
@@ -372,7 +375,7 @@ class AudioDecoder(Model):
 
         loss_stft = self.audio_processor.stft_loss(fake_audio, real_audio) * stft_scale
         loss_mel = (
-            F.l1_loss(self.audio_processor.compute_mel(fake_audio), mels) * mel_scale
+            F.huber_loss(self.audio_processor.compute_mel(fake_audio), mels) * mel_scale
         )
         loss_fm = ((loss_fm_mpd + loss_fm_msd) * fm_scale) + fm_add
 
@@ -380,9 +383,10 @@ class AudioDecoder(Model):
 
         loss_g = loss_adv + loss_fm + loss_stft + loss_mel
         if not am_i_frozen:
-            self.g_optim.zero_grad()
             loss_g.backward()
             self.g_optim.step()
+
+        lr_g, lr_d = self.get_lr()
         return {
             "loss_g": loss_g.item(),
             "loss_d": loss_d,
@@ -390,8 +394,8 @@ class AudioDecoder(Model):
             "loss_fm": loss_fm.item(),
             "loss_stft": loss_stft.item(),
             "loss_mel": loss_mel.item(),
-            "lr_g": self.g_optim.param_groups[0]["lr"],
-            "lr_d": self.d_optim.param_groups[0]["lr"],
+            "lr_g": lr_g,
+            "lr_d": lr_d,
         }
 
     def step_scheduler(
@@ -417,34 +421,198 @@ class AudioDecoder(Model):
             self.g_scheduler = self.settings.scheduler_template(self.g_optim)
 
 
-class ResBlocks(ConvNets):
+class AudioGeneratorOnlyTrainer(Model):
     def __init__(
         self,
-        channels: int,
-        resblock_kernel_sizes: List[Union[int, List[int]]] = [3, 7, 11],
-        resblock_dilation_sizes: List[Union[int, List[int]]] = [
-            [1, 3, 5],
-            [1, 3, 5],
-            [1, 3, 5],
-        ],
-        activation: nn.Module = nn.LeakyReLU(0.1),
+        audio_processor: AudioProcessor,
+        settings: Optional[AudioSettings] = None,
+        generator: Optional[Union[Model, "iSTFTGenerator"]] = None,  # non initialized!
     ):
         super().__init__()
-        self.num_kernels = len(resblock_kernel_sizes)
-        self.rb = nn.ModuleList()
-        self.activation = activation
+        if settings is None:
+            self.settings = AudioSettings()
+        elif isinstance(settings, dict):
+            self.settings = AudioSettings(**settings)
+        elif isinstance(settings, AudioSettings):
+            self.settings = settings
+        else:
+            raise ValueError(
+                "Cannot initialize the waveDecoder with the given settings. "
+                "Use either a dictionary, or the class WaveSettings to setup the settings. "
+                "Alternatively, leave it None to use the default values."
+            )
+        if self.settings.seed is not None:
+            set_seed(self.settings.seed)
+        if generator is None:
+            generator = iSTFTGenerator
+        self.generator: iSTFTGenerator = generator(
+            in_channels=self.settings.in_channels,
+            upsample_rates=self.settings.upsample_rates,
+            upsample_kernel_sizes=self.settings.upsample_kernel_sizes,
+            upsample_initial_channel=self.settings.upsample_initial_channel,
+            resblock_kernel_sizes=self.settings.resblock_kernel_sizes,
+            resblock_dilation_sizes=self.settings.resblock_dilation_sizes,
+            n_fft=self.settings.n_fft,
+            activation=self.settings.activation,
+        )
+        self.generator.eval()
+        self.gen_training = False
+        self.audio_processor = audio_processor
 
-        for k, j in zip(resblock_kernel_sizes, resblock_dilation_sizes):
-            self.rb.append(ResBlock1D(channels, k, j, activation))
+    def setup_training_mode(self, *args, **kwargs):
+        self.finish_training_setup()
+        self.update_schedulers_and_optimizer()
+        self.gen_training = True
+        return True
+
+    def update_schedulers_and_optimizer(self):
+        self.g_optim = optim.AdamW(
+            self.generator.parameters(),
+            lr=self.settings.lr,
+            betas=self.settings.adamw_betas,
+        )
+        self.g_scheduler = self.settings.scheduler_template(self.g_optim)
 
-        self.rb.apply(self.init_weights)
+    def set_lr(self, new_lr: float = 1e-4):
+        if self.g_optim is not None:
+            for groups in self.g_optim.param_groups:
+                groups["lr"] = new_lr
+        return self.get_lr()
 
-    def forward(self, x: torch.Tensor):
-        xs = None
-        for i, block in enumerate(self.rb):
-            if i == 0:
-                xs = block(x)
-            else:
-                xs += block(x)
-        x = xs / self.num_kernels
-        return self.activation(x)
+    def get_lr(self) -> Tuple[float, float]:
+        if self.g_optim is not None:
+            return self.g_optim.param_groups[0]["lr"]
+        return float("nan")
+
+    def save_weights(self, path, replace=True):
+        is_pathlike(path, check_if_empty=True, validate=True)
+        if str(path).endswith(".pt"):
+            path = Path(path).parent
+        else:
+            path = Path(path)
+        self.generator.save_weights(Path(path, "generator.pt"), replace)
+
+    def load_weights(
+        self,
+        path,
+        raise_if_not_exists=False,
+        strict=True,
+        assign=False,
+        weights_only=False,
+        mmap=None,
+        **torch_loader_kwargs
+    ):
+        is_pathlike(path, check_if_empty=True, validate=True)
+        if str(path).endswith(".pt"):
+            path = Path(path)
+        else:
+            path = Path(path, "generator.pt")
+
+        self.generator.load_weights(
+            path,
+            raise_if_not_exists,
+            strict,
+            assign,
+            weights_only,
+            mmap,
+            **torch_loader_kwargs,
+        )
+
+    def finish_training_setup(self):
+        gc.collect()
+        clear_cache()
+        self.eval()
+        self.gen_training = False
+
+    def forward(self, mel_spec: Tensor) -> Tuple[Tensor, Tensor]:
+        """Returns the generated spec and phase"""
+        return self.generator.forward(mel_spec)
+
+    def inference(
+        self,
+        mel_spec: Tensor,
+        return_dict: bool = False,
+    ) -> Union[Dict[str, Tensor], Tensor]:
+        spec, phase = super().inference(mel_spec)
+        wave = self.audio_processor.inverse_transform(
+            spec,
+            phase,
+            self.settings.n_fft,
+            hop_length=4,
+            win_length=self.settings.n_fft,
+        )
+        if not return_dict:
+            return wave[:, : wave.shape[-1] - 256]
+        return {
+            "wave": wave[:, : wave.shape[-1] - 256],
+            "spec": spec,
+            "phase": phase,
+        }
+
+    def set_device(self, device: str):
+        self.to(device=device)
+        self.generator.to(device=device)
+        self.audio_processor.to(device=device)
+        self.msd.to(device=device)
+        self.mpd.to(device=device)
+
+    def train_step(
+        self,
+        mels: Tensor,
+        real_audio: Tensor,
+        stft_scale: float = 1.0,
+        mel_scale: float = 1.0,
+        ext_loss: Optional[Callable[[Tensor, Tensor], Tensor]] = None,
+    ):
+        if not self.gen_training:
+            self.setup_training_mode()
+
+        self.g_optim.zero_grad()
+        spec, phase = self.generator.train_step(mels)
+
+        real_audio = real_audio.squeeze(1)
+        with torch.no_grad():
+            fake_audio = self.audio_processor.inverse_transform(
+                spec,
+                phase,
+                self.settings.n_fft,
+                hop_length=4,
+                win_length=self.settings.n_fft,
+            )[:, : real_audio.shape[-1]]
+        loss_stft = self.audio_processor.stft_loss(fake_audio, real_audio) * stft_scale
+        loss_mel = (
+            F.huber_loss(self.audio_processor.compute_mel(fake_audio), mels) * mel_scale
+        )
+        loss_g.backward()
+        loss_g = loss_stft + loss_mel
+        loss_ext = 0
+
+        if ext_loss is not None:
+            l_ext = ext_loss(fake_audio, real_audio)
+            loss_g = loss_g + l_ext
+            loss_ext = l_ext.item()
+
+        self.g_optim.step()
+        return {
+            "loss": loss_g.item(),
+            "loss_stft": loss_stft.item(),
+            "loss_mel": loss_mel.item(),
+            "loss_ext": loss_ext,
+            "lr": self.get_lr(),
+        }
+
+    def step_scheduler(self):
+
+        if self.g_scheduler is not None:
+            self.g_scheduler.step()
+
+    def reset_schedulers(self, lr: Optional[float] = None):
+        """
+        In case you have adopted another strategy, with this function,
+        it is possible restart the scheduler and set the lr to another value.
+        """
+        if lr is not None:
+            self.set_lr(lr)
+        if self.g_optim is not None:
+            self.g_scheduler = None
+            self.g_scheduler = self.settings.scheduler_template(self.g_optim)